diff --git a/swh/loader/pypi/client.py b/swh/loader/pypi/client.py index 5467382e7391ea912fbabe29013ac5bbfb646fcc..a7c54fbf8d58ce52f8da83f52e12601080909231 100644 --- a/swh/loader/pypi/client.py +++ b/swh/loader/pypi/client.py @@ -48,12 +48,7 @@ class PyPiClient: """ def __init__(self, temp_directory=None, cache=False, cache_dir=None): self.version = __version__ - if not temp_directory: - from tempfile import mkdtemp - self.temp_directory = mkdtemp(dir=temp_directory, - prefix='swh.loader.pypi.client') - else: - self.temp_directory = temp_directory + self.temp_directory = temp_directory self.do_cache = cache if self.do_cache: diff --git a/swh/loader/pypi/loader.py b/swh/loader/pypi/loader.py index 96ec9c635feffb1d93c263b4b1ae37b65895b966..75bb51eeab6777b2a0ad3ddc20c3f08e9227bbb4 100644 --- a/swh/loader/pypi/loader.py +++ b/swh/loader/pypi/loader.py @@ -8,6 +8,8 @@ import logging import os import shutil +from tempfile import mkdtemp + from swh.loader.core.utils import clean_dangling_folders from swh.loader.core.loader import SWHLoader from swh.model.from_disk import Directory @@ -21,7 +23,7 @@ from .model import PyPiProject TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.pypi.' - +DEBUG_MODE = '** DEBUG MODE **' class PyPiLoader(SWHLoader): CONFIG_BASE_FILENAME = 'loader/pypi' @@ -35,7 +37,13 @@ class PyPiLoader(SWHLoader): def __init__(self): super().__init__(logging_class='swh.loader.pypi.PyPiLoader') self.origin_id = None - self.temp_directory = self.config['temp_directory'] + + temp_directory = self.config['temp_directory'] + os.makedirs(temp_directory, exist_ok=True) + self.temp_directory = mkdtemp( + suffix='-%s' % os.getpid(), + prefix=TEMPORARY_DIR_PREFIX_PATTERN, + dir=temp_directory) self.pypi_client = PyPiClient( temp_directory=self.temp_directory, cache=self.config['cache'], @@ -47,7 +55,12 @@ class PyPiLoader(SWHLoader): in mid-air (OOM killed), we try and clean up dangling files. """ - clean_dangling_folders(self.temp_directory, + if self.debug: + self.log.warn('%s Will not pre-clean up temp dir %s' % ( + DEBUG_MODE, self.temp_directory + )) + return + clean_dangling_folders(self.config['temp_directory'], pattern_check=TEMPORARY_DIR_PREFIX_PATTERN, log=self.log) @@ -56,8 +69,8 @@ class PyPiLoader(SWHLoader): """ if self.debug: - self.log.warn('** DEBUG MODE ** Will not clean up temp dir %s' % ( - self.temp_directory + self.log.warn('%s Will not clean up temp dir %s' % ( + DEBUG_MODE, self.temp_directory )) return if os.path.exists(self.temp_directory):