PKG-INFO

+Metadata-Version: 1.0
+Name: swh.model
+Version: 0.0.8
+Summary: Software Heritage data model
+Home-page: https://forge.softwareheritage.org/diffusion/DMOD/
+Author: Software Heritage developers
+Author-email: swh-devel@inria.fr
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

debian/changelog

-swh-model (0.0.1-1) unstable; urgency=low
+swh-model (0.0.7-1~swh1) unstable-swh; urgency=medium
 
- * Create swh-model package
+  * v0.0.7
+  * Improve corner case policy about walking and computing hash tree (+
+    update)
 
- -- Nicolas Dandrimont <olasd@debian.org>  Mon, 07 Dec 2015 15:41:28 +0100
+ -- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com>  Wed, 25 May 2016 23:47:19 +0200
+
+swh-model (0.0.6-1~swh1) unstable-swh; urgency=medium
+
+  * v0.0.6
+  * Improve corner case on git hash memory update function
+  * debian packaging: Ignore fs tests for packaging
+
+ -- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com>  Tue, 24 May 2016 17:01:06 +0200
+
+swh-model (0.0.5-1~swh1) unstable-swh; urgency=medium
+
+  * v0.0.5
+  * Add update git hash computation from existing data
+  * Add revision identifier data for hash identifier computation (extra-
+    headers)
+
+ -- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com>  Fri, 15 Apr 2016 12:51:21 +0200
+
+swh-model (0.0.4-1~swh1) unstable-swh; urgency=medium
+
+  * v0.0.4
+  * Migrate swh.loader.dir.git module to swh.model.git
+
+ -- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com>  Mon, 21 Mar 2016 15:20:28 +0100
+
+swh-model (0.0.3-1~swh1) unstable-swh; urgency=medium
+
+  * v0.0.3
+  * Release name is now in bytes
+
+ -- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com>  Wed, 27 Jan 2016 15:50:08 +0100
+
+swh-model (0.0.2-1~swh1) unstable-swh; urgency=medium
+
+  * Prepare release of v0.0.2
+  * Import the rest of swh.core.hashutil
+
+ -- Nicolas Dandrimont <nicolas@dandrimont.eu>  Wed, 16 Dec 2015 18:30:12 +0100
+
+swh-model (0.0.1-1~swh1) unstable-swh; urgency=medium
+
+  * Initial release
+  * Prepare swh.model release v0.0.1
+
+ -- Nicolas Dandrimont <nicolas@dandrimont.eu>  Mon, 07 Dec 2015 18:26:58 +0100

debian/control

@@ -3,9 +3,11 @@ Maintainer: Software Heritage developers <swh-devel@inria.fr>
 Section: python
 Priority: optional
 Build-Depends: debhelper (>= 9),
-               dh-python,
+               dh-python (>= 2),
+               python3 (>= 3.5) | python3-pyblake2,
                python3-all,
-               python3-nose,
+               python3-click,
+               python3-pytest,
                python3-setuptools,
                python3-vcversioner
 Standards-Version: 3.9.6
@@ -14,4 +16,7 @@ Homepage: https://forge.softwareheritage.org/diffusion/DMOD/
 Package: python3-swh.model
 Architecture: all
 Depends: ${misc:Depends}, ${python3:Depends}
+Breaks: python3-swh.loader.core (<< 0.0.16~),
+        python3-swh.loader.dir (<< 0.0.28~),
+        python3-swh.loader.svn (<< 0.0.28~)
 Description: Software Heritage data model

debian/gbp.conf

+[DEFAULT]
+upstream-branch=debian/upstream
+upstream-tag=debian/upstream/%(version)s
+upstream-vcs-tag=v%(version)s
+debian-branch=debian/unstable-swh
+pristine-tar=True

debian/rules

 #!/usr/bin/make -f
 
-export PYBUILD_NAME=swh-model
+export PYBUILD_NAME=swh.model
+export export PYBUILD_TEST_ARGS=-m 'not db and not fs'
 
 %:
 	dh $@ --with python3 --buildsystem=pybuild
 
-override_dh_auto_test:
-	PYBUILD_SYSTEM=custom \
-        PYBUILD_TEST_ARGS="python{version} -m nose swh -sva '!db,!fs'" \
-        dh_auto_test
+override_dh_install:
+	dh_install
+	rm -v $(CURDIR)/debian/python3-*/usr/lib/python*/dist-packages/swh/__init__.py

setup.cfg

+[egg_info]
+tag_build = 
+tag_date = 0
+

swh.model.egg-info/PKG-INFO

+Metadata-Version: 1.0
+Name: swh.model
+Version: 0.0.8
+Summary: Software Heritage data model
+Home-page: https://forge.softwareheritage.org/diffusion/DMOD/
+Author: Software Heritage developers
+Author-email: swh-devel@inria.fr
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

swh.model.egg-info/SOURCES.txt

+.gitignore
+AUTHORS
+LICENSE
+MANIFEST.in
+Makefile
+Makefile.local
+README-dev.md
+requirements.txt
+setup.py
+version.txt
+debian/changelog
+debian/compat
+debian/control
+debian/copyright
+debian/rules
+debian/source/format
+swh.model.egg-info/PKG-INFO
+swh.model.egg-info/SOURCES.txt
+swh.model.egg-info/dependency_links.txt
+swh.model.egg-info/requires.txt
+swh.model.egg-info/top_level.txt
+swh/model/__init__.py
+swh/model/exceptions.py
+swh/model/git.py
+swh/model/hashutil.py
+swh/model/identifiers.py
+swh/model/validators.py
+swh/model/fields/__init__.py
+swh/model/fields/compound.py
+swh/model/fields/hashes.py
+swh/model/fields/simple.py
+swh/model/tests/__init__.py
+swh/model/tests/test_git.py
+swh/model/tests/test_git_slow.py
+swh/model/tests/test_hashutil.py
+swh/model/tests/test_identifiers.py
+swh/model/tests/test_validators.py
+swh/model/tests/fields/__init__.py
+swh/model/tests/fields/test_compound.py
+swh/model/tests/fields/test_hashes.py
+swh/model/tests/fields/test_simple.py
\ No newline at end of file

swh.model.egg-info/dependency_links.txt

+

swh.model.egg-info/requires.txt

+vcversioner

swh.model.egg-info/top_level.txt

+swh

swh/model/git.py

@@ -186,8 +186,116 @@ def compute_tree_metadata(dirname, ls_hashes):
     }
 
 
+def default_validation_dir(dirpath):
+    """Default validation function.
+       This is the equivalent of the identity function.
+
+    Args:
+        dirpath: Path to validate
+
+    Returns: True
+
+    """
+    return True
+
+
+def __walk(rootdir,
+           dir_ok_fn=default_validation_dir,
+           remove_empty_folder=False):
+    """Walk the filesystem and yields a 3 tuples (dirpath, dirnames as set
+    of absolute paths, filenames as set of abslute paths)
+
+       Ignore files which won't pass the dir_ok_fn validation.
+
+       If remove_empty_folder is True, remove and ignore any
+       encountered empty folder.
+
+    Args:
+        - rootdir: starting walk root directory path
+        - dir_ok_fn: validation function. if folder encountered are
+        not ok, they are ignored.  Default to default_validation_dir
+        which does nothing.
+         - remove_empty_folder: Flag to remove and ignore any
+          encountered empty folders.
+
+    Yields:
+        3 tuples dirpath, set of absolute children dirname paths, set
+        of absolute filename paths.
+
+    """
+    def basic_gen_dir(rootdir):
+        for dp, dns, fns in os.walk(rootdir, topdown=False):
+            yield (dp,
+                   set((os.path.join(dp, dn) for dn in dns)),
+                   set((os.path.join(dp, fn) for fn in fns)))
+
+    if dir_ok_fn == default_validation_dir:
+        if not remove_empty_folder:  # os.walk
+            yield from basic_gen_dir(rootdir)
+        else:                        # os.walk + empty dir cleanup
+            empty_folders = set()
+            for dp, dns, fns in basic_gen_dir(rootdir):
+                if not dns and not fns:
+                    empty_folders.add(dp)
+                    # need to remove it because folder of empty folder
+                    # is an empty folder!!!
+                    if os.path.islink(dp):
+                        os.remove(dp)
+                    else:
+                        os.rmdir(dp)
+                    parent = os.path.dirname(dp)
+                    # edge case about parent containing one empty
+                    # folder which become an empty one
+                    while not os.listdir(parent):
+                        empty_folders.add(parent)
+                        if os.path.islink(parent):
+                            os.remove(parent)
+                        else:
+                            os.rmdir(parent)
+                        parent = os.path.dirname(parent)
+                    continue
+                yield (dp, dns - empty_folders, fns)
+    else:
+        def filtfn(dirnames):
+            return set(filter(dir_ok_fn, dirnames))
+
+        gen_dir = ((dp, dns, fns) for dp, dns, fns
+                   in basic_gen_dir(rootdir) if dir_ok_fn(dp))
+
+        if not remove_empty_folder:  # os.walk + filtering
+            for dp, dns, fns in gen_dir:
+                yield (dp, filtfn(dns), fns)
+        else:                        # os.walk + filtering + empty dir cleanup
+            empty_folders = set()
+            for dp, dns, fns in gen_dir:
+                dps = filtfn(dns)
+
+                if not dps and not fns:
+                    empty_folders.add(dp)
+                    # need to remove it because folder of empty folder
+                    # is an empty folder!!!
+                    if os.path.islink(dp):
+                        print('remove link to empty folder')
+                        os.remove(dp)
+                    else:
+                        print('remove empty folder')
+                        os.rmdir(dp)
+                    parent = os.path.dirname(dp)
+                    # edge case about parent containing one empty
+                    # folder which become an empty one
+                    while not os.listdir(parent):
+                        empty_folders.add(parent)
+                        if os.path.islink(parent):
+                            os.remove(parent)
+                        else:
+                            os.rmdir(parent)
+                        parent = os.path.dirname(parent)
+                    continue
+                yield dp, dps - empty_folders, fns
+
+
 def walk_and_compute_sha1_from_directory(rootdir,
-                                         dir_ok_fn=lambda dirpath: True,
+                                         dir_ok_fn=default_validation_dir,
                                          with_root_tree=True,
                                          remove_empty_folder=False):
     """Compute git sha1 from directory rootdir.
@@ -228,53 +336,27 @@ def walk_and_compute_sha1_from_directory(rootdir,
     if rootdir.endswith(b'/'):
         rootdir = rootdir.rstrip(b'/')
 
-    def filtfn(dirpath, dirnames):
-        return list(filter(lambda dirname: dir_ok_fn(os.path.join(dirpath,
-                                                                  dirname)),
-                           dirnames))
-
-    if remove_empty_folder:  # round-trip to remove empty folders
-        gen_dir = ((dp, filtfn(dp, dns), fns) for (dp, dns, fns)
-                   in os.walk(rootdir, topdown=False)
-                   if dir_ok_fn(dp))
-        for dirpath, dirnames, filenames in gen_dir:
-            if dirnames == [] and filenames == []:
-                if os.path.islink(dirpath):
-                    os.remove(dirpath)
-                else:
-                    os.removedirs(dirpath)
-
-    gen_dir = ((dp, filtfn(dp, dns), fns) for (dp, dns, fns)
-               in os.walk(rootdir, topdown=False)
-               if dir_ok_fn(dp))
-
-    for dirpath, dirnames, filenames in gen_dir:
+    for dirpath, dirnames, filenames in __walk(
+            rootdir, dir_ok_fn, remove_empty_folder):
         hashes = []
 
-        links = (os.path.join(dirpath, file)
-                 for file in (filenames+dirnames)
-                 if os.path.islink(os.path.join(dirpath, file)))
+        links = (file
+                 for file in filenames.union(dirnames)
+                 if os.path.islink(file))
 
         for linkpath in links:
             all_links.add(linkpath)
             m_hashes = compute_link_metadata(linkpath)
             hashes.append(m_hashes)
 
-        only_files = (os.path.join(dirpath, file)
-                      for file in filenames
-                      if os.path.join(dirpath, file) not in all_links)
-        for filepath in only_files:
+        for filepath in (file for file in filenames if file not in all_links):
             m_hashes = compute_blob_metadata(filepath)
             hashes.append(m_hashes)
 
         ls_hashes[dirpath] = hashes
 
         dir_hashes = []
-        subdirs = (os.path.join(dirpath, dir)
-                   for dir in dirnames
-                   if os.path.join(dirpath, dir)
-                   not in all_links)
-        for fulldirname in subdirs:
+        for fulldirname in (dir for dir in dirnames if dir not in all_links):
             tree_hash = compute_tree_metadata(fulldirname, ls_hashes)
             dir_hashes.append(tree_hash)
 
@@ -398,7 +480,7 @@ def commonpath(paths):
 
 
 def __remove_paths_from_objects(objects, rootpaths,
-                                dir_ok_fn=lambda dirpath: True):
+                                dir_ok_fn=default_validation_dir):
     """Given top paths to remove, remove all paths and descendants from
     objects.
 
@@ -441,7 +523,7 @@ def __remove_paths_from_objects(objects, rootpaths,
 
 
 def update_checksums_from(changed_paths, objects,
-                          dir_ok_fn=lambda dirpath: True,
+                          dir_ok_fn=default_validation_dir,
                           remove_empty_folder=False):
     """Given a list of changed paths, recompute the checksums only where
     needed.

version.txt

+v0.0.8-0-gd124e6e
\ No newline at end of file