Refactor how offices files are handled

- xml files are no longer considered harmless
- Factorization of the `remove_all` method for office files
- Explicit whitelist are used
- Blacklist are used to skip files completely
  - Non-blacklisted files are _still cleaned_
  - Unsupported files are still triggering an error

libmat2/harmless.py

@@ -4,7 +4,7 @@ from . import abstract
 
 class HarmlessParser(abstract.AbstractParser):
     """ This is the parser for filetypes that do not contain metadata. """
-    mimetypes = {'application/xml', 'text/plain', 'text/xml', 'application/rdf+xml'}
+    mimetypes = {'text/plain', }
 
     def __init__(self, filename: str) -> None:
         super().__init__(filename)

libmat2/office.py

@@ -4,17 +4,16 @@ import shutil
 import tempfile
 import datetime
 import zipfile
-from typing import Dict, Set
+from typing import Dict, Set, Pattern
 
 from . import abstract, parser_factory
 
-assert Set   # make pyflakes happy
 
 class ArchiveBasedAbstractParser(abstract.AbstractParser):
-    whitelist = set()  # type: Set[str]
+    files_to_keep : Set[str] = set()
+    files_to_omit : Set[Pattern] = set()
 
     def _clean_zipinfo(self, zipinfo: zipfile.ZipInfo) -> zipfile.ZipInfo:
-        zipinfo.compress_type = zipfile.ZIP_DEFLATED
         zipinfo.create_system = 3  # Linux
         zipinfo.comment = b''
         zipinfo.date_time = (1980, 1, 1, 0, 0, 0)
@@ -34,33 +33,51 @@ class ArchiveBasedAbstractParser(abstract.AbstractParser):
             metadata['comment'] = zipinfo.comment  # type: ignore
 
         if zipinfo.date_time != (1980, 1, 1, 0, 0, 0):
-            metadata['date_time'] =str(datetime.datetime(*zipinfo.date_time))
+            metadata['date_time'] = str(datetime.datetime(*zipinfo.date_time))
 
         return metadata
 
 
     def _clean_internal_file(self, item: zipfile.ZipInfo, temp_folder: str,
                              zin: zipfile.ZipFile, zout: zipfile.ZipFile) -> bool:
-        output = ''
         zin.extract(member=item, path=temp_folder)
-        if item.filename not in self.whitelist:
+        full_path = os.path.join(temp_folder, item.filename)
-            full_path = os.path.join(temp_folder, item.filename)
+        tmp_parser, mtype = parser_factory.get_parser(full_path)  # type: ignore
-            tmp_parser, mtype = parser_factory.get_parser(full_path)  # type: ignore
+        if not tmp_parser:
-            if not tmp_parser:
+            zout.close()
-                zout.close()
+            os.remove(self.output_filename)
-                os.remove(self.output_filename)
+            print("%s's format (%s) isn't supported" % (item.filename, mtype))
-                print("%s's format (%s) isn't supported" % (item.filename, mtype))
+            return False
-                return False
+        tmp_parser.remove_all()
-            tmp_parser.remove_all()
+
-            output = tmp_parser.output_filename
-        else:
-            output = os.path.join(temp_folder, item.filename)
         zinfo = zipfile.ZipInfo(item.filename)  # type: ignore
         clean_zinfo = self._clean_zipinfo(zinfo)
-        with open(output, 'rb') as f:
+        with open(tmp_parser.output_filename, 'rb') as f:
             zout.writestr(clean_zinfo, f.read())
         return True
 
+    def remove_all(self) -> bool:
+        zin = zipfile.ZipFile(self.filename, 'r')
+        zout = zipfile.ZipFile(self.output_filename, 'w')
+        temp_folder = tempfile.mkdtemp()
+
+        for item in zin.infolist():
+            if item.filename[-1] == '/':  # `is_dir` is added in Python3.6
+                continue  # don't keep empty folders
+            elif item.filename in self.files_to_keep:
+                item = self._clean_zipinfo(item)
+                zout.writestr(item, zin.read(item))
+                continue
+            elif any(map(lambda r: r.search(item.filename), self.files_to_omit)):
+                continue
+            elif not self._clean_internal_file(item, temp_folder, zin, zout):
+                return False
+
+        shutil.rmtree(temp_folder)
+        zout.close()
+        zin.close()
+        return True
+
 
 class MSOfficeParser(ArchiveBasedAbstractParser):
     mimetypes = {
@@ -68,9 +85,20 @@ class MSOfficeParser(ArchiveBasedAbstractParser):
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
         'application/vnd.openxmlformats-officedocument.presentationml.presentation'
     }
-    files_to_keep = {'_rels/.rels', 'word/_rels/document.xml.rels'}
+    files_to_keep = {
+            '[Content_Types].xml',
+            '_rels/.rels',
+            'word/_rels/document.xml.rels',
+            'word/document.xml',
+            'word/fontTable.xml',
+            'word/settings.xml',
+            'word/styles.xml',
+    }
+    files_to_omit = set(map(re.compile, {  # type: ignore
+            '^docProps/',
+    }))
 
-    def get_meta(self):
+    def get_meta(self) -> Dict[str, str]:
         """
         Yes, I know that parsing xml with regexp ain't pretty,
         be my guest and fix it if you want.
@@ -88,38 +116,12 @@ class MSOfficeParser(ArchiveBasedAbstractParser):
                     pass
                 if not metadata:  # better safe than sorry
                     metadata[item] = 'harmful content'
-
             for key, value in self._get_zipinfo_meta(item).items():
                 metadata[key] = value
         zipin.close()
         return metadata
 
 
-    def remove_all(self):
-        zin = zipfile.ZipFile(self.filename, 'r')
-        zout = zipfile.ZipFile(self.output_filename, 'w')
-        temp_folder = tempfile.mkdtemp()
-
-        for item in zin.infolist():
-            if item.filename[-1] == '/':
-                continue  # `is_dir` is added in Python3.6
-            elif item.filename.startswith('docProps/'):
-                continue  # don't keep metadata files
-            if item.filename in self.files_to_keep:
-                item = self._clean_zipinfo(item)
-                zout.writestr(item, zin.read(item))
-                continue
-
-            if self._clean_internal_file(item, temp_folder, zin, zout) is False:
-                return False
-
-        shutil.rmtree(temp_folder)
-        zout.close()
-        zin.close()
-        return True
-
-
-
 class LibreOfficeParser(ArchiveBasedAbstractParser):
     mimetypes = {
         'application/vnd.oasis.opendocument.text',
@@ -130,10 +132,20 @@ class LibreOfficeParser(ArchiveBasedAbstractParser):
         'application/vnd.oasis.opendocument.formula',
         'application/vnd.oasis.opendocument.image',
     }
-    whitelist = {'mimetype', 'manifest.rdf'}
+    files_to_keep = {
+            'META-INF/manifest.xml',
+            'content.xml',
+            'manifest.rdf',
+            'mimetype',
+            'settings.xml',
+            'styles.xml',
+    }
+    files_to_omit = set(map(re.compile, {  # type: ignore
+            '^meta\.xml$',
+            '^Configurations2/',
+    }))
 
-
+    def get_meta(self) -> Dict[str, str]:
-    def get_meta(self):
         """
         Yes, I know that parsing xml with regexp ain't pretty,
         be my guest and fix it if you want.
@@ -156,21 +168,3 @@ class LibreOfficeParser(ArchiveBasedAbstractParser):
         zipin.close()
         return metadata
 
-    def remove_all(self):
-        zin = zipfile.ZipFile(self.filename, 'r')
-        zout = zipfile.ZipFile(self.output_filename, 'w')
-        temp_folder = tempfile.mkdtemp()
-
-        for item in zin.infolist():
-            if item.filename[-1] == '/':
-                continue  # `is_dir` is added in Python3.6
-            elif item.filename == 'meta.xml':
-                continue  # don't keep metadata files
-
-            if self._clean_internal_file(item, temp_folder, zin, zout) is False:
-                return False
-
-        shutil.rmtree(temp_folder)
-        zout.close()
-        zin.close()
-        return True