swh.loader.pypi: Improve temporary directory bootstrap

Related T421

swh/loader/pypi/client.py

@@ -48,12 +48,7 @@ class PyPiClient:
     """
     def __init__(self, temp_directory=None, cache=False, cache_dir=None):
         self.version = __version__
-        if not temp_directory:
-            from tempfile import mkdtemp
-            self.temp_directory = mkdtemp(dir=temp_directory,
-                                          prefix='swh.loader.pypi.client')
-        else:
-            self.temp_directory = temp_directory
+        self.temp_directory = temp_directory
 
         self.do_cache = cache
         if self.do_cache:

swh/loader/pypi/loader.py

@@ -8,6 +8,8 @@ import logging
 import os
 import shutil
 
+from tempfile import mkdtemp
+
 from swh.loader.core.utils import clean_dangling_folders
 from swh.loader.core.loader import SWHLoader
 from swh.model.from_disk import Directory
@@ -21,7 +23,7 @@ from .model import PyPiProject
 
 
 TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.pypi.'
-
+DEBUG_MODE = '** DEBUG MODE **'
 
 class PyPiLoader(SWHLoader):
     CONFIG_BASE_FILENAME = 'loader/pypi'
@@ -35,7 +37,13 @@ class PyPiLoader(SWHLoader):
     def __init__(self):
         super().__init__(logging_class='swh.loader.pypi.PyPiLoader')
         self.origin_id = None
-        self.temp_directory = self.config['temp_directory']
+
+        temp_directory = self.config['temp_directory']
+        os.makedirs(temp_directory, exist_ok=True)
+        self.temp_directory = mkdtemp(
+            suffix='-%s' % os.getpid(),
+            prefix=TEMPORARY_DIR_PREFIX_PATTERN,
+            dir=temp_directory)
         self.pypi_client = PyPiClient(
             temp_directory=self.temp_directory,
             cache=self.config['cache'],
@@ -47,7 +55,12 @@ class PyPiLoader(SWHLoader):
         in mid-air (OOM killed), we try and clean up dangling files.
 
         """
-        clean_dangling_folders(self.temp_directory,
+        if self.debug:
+            self.log.warn('%s Will not pre-clean up temp dir %s' % (
+                DEBUG_MODE, self.temp_directory
+            ))
+            return
+        clean_dangling_folders(self.config['temp_directory'],
                                pattern_check=TEMPORARY_DIR_PREFIX_PATTERN,
                                log=self.log)
 
@@ -56,8 +69,8 @@ class PyPiLoader(SWHLoader):
 
         """
         if self.debug:
-            self.log.warn('** DEBUG MODE ** Will not clean up temp dir  %s' % (
-                self.temp_directory
+            self.log.warn('%s Will not clean up temp dir %s' % (
+                DEBUG_MODE, self.temp_directory
             ))
             return
         if os.path.exists(self.temp_directory):

[Phabricator Migration user]

Some references in the commit message have been migrated:

• T421 is now swh-loader-core#421 (closed)