From 9f28590b261c8f8e2384a6a99c105faab3f231bc Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <antoine.romain.dumont@gmail.com>
Date: Fri, 3 Aug 2018 13:25:13 +0200
Subject: [PATCH] swh.loader.pypi: Improve temporary directory bootstrap
Related T421
---
swh/loader/pypi/client.py | 7 +------
swh/loader/pypi/loader.py | 23 ++++++++++++++++++-----
2 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/swh/loader/pypi/client.py b/swh/loader/pypi/client.py
index 5467382..a7c54fb 100644
--- a/swh/loader/pypi/client.py
+++ b/swh/loader/pypi/client.py
@@ -48,12 +48,7 @@ class PyPiClient:
"""
def __init__(self, temp_directory=None, cache=False, cache_dir=None):
self.version = __version__
- if not temp_directory:
- from tempfile import mkdtemp
- self.temp_directory = mkdtemp(dir=temp_directory,
- prefix='swh.loader.pypi.client')
- else:
- self.temp_directory = temp_directory
+ self.temp_directory = temp_directory
self.do_cache = cache
if self.do_cache:
diff --git a/swh/loader/pypi/loader.py b/swh/loader/pypi/loader.py
index 96ec9c6..75bb51e 100644
--- a/swh/loader/pypi/loader.py
+++ b/swh/loader/pypi/loader.py
@@ -8,6 +8,8 @@ import logging
import os
import shutil
+from tempfile import mkdtemp
+
from swh.loader.core.utils import clean_dangling_folders
from swh.loader.core.loader import SWHLoader
from swh.model.from_disk import Directory
@@ -21,7 +23,7 @@ from .model import PyPiProject
TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.pypi.'
-
+DEBUG_MODE = '** DEBUG MODE **'
class PyPiLoader(SWHLoader):
CONFIG_BASE_FILENAME = 'loader/pypi'
@@ -35,7 +37,13 @@ class PyPiLoader(SWHLoader):
def __init__(self):
super().__init__(logging_class='swh.loader.pypi.PyPiLoader')
self.origin_id = None
- self.temp_directory = self.config['temp_directory']
+
+ temp_directory = self.config['temp_directory']
+ os.makedirs(temp_directory, exist_ok=True)
+ self.temp_directory = mkdtemp(
+ suffix='-%s' % os.getpid(),
+ prefix=TEMPORARY_DIR_PREFIX_PATTERN,
+ dir=temp_directory)
self.pypi_client = PyPiClient(
temp_directory=self.temp_directory,
cache=self.config['cache'],
@@ -47,7 +55,12 @@ class PyPiLoader(SWHLoader):
in mid-air (OOM killed), we try and clean up dangling files.
"""
- clean_dangling_folders(self.temp_directory,
+ if self.debug:
+ self.log.warn('%s Will not pre-clean up temp dir %s' % (
+ DEBUG_MODE, self.temp_directory
+ ))
+ return
+ clean_dangling_folders(self.config['temp_directory'],
pattern_check=TEMPORARY_DIR_PREFIX_PATTERN,
log=self.log)
@@ -56,8 +69,8 @@ class PyPiLoader(SWHLoader):
"""
if self.debug:
- self.log.warn('** DEBUG MODE ** Will not clean up temp dir %s' % (
- self.temp_directory
+ self.log.warn('%s Will not clean up temp dir %s' % (
+ DEBUG_MODE, self.temp_directory
))
return
if os.path.exists(self.temp_directory):
--
GitLab