Skip to content
Snippets Groups Projects
Verified Commit 5b652b30 authored by Antoine R. Dumont's avatar Antoine R. Dumont
Browse files

lister.debian: Make debian init step idempotent and up-to-date

parent 4b9f0e05
No related branches found
Tags v0.0.46
1 merge request!132lister.debian: Refactor and update the debian initialization step
......@@ -3,46 +3,62 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import logging
from typing import Any, List, Mapping
logger = logging.getLogger(__name__)
def debian_init(db_engine,
override_conf: Mapping[str, Any] = {},
distributions: List[str] = ['stretch', 'buster'],
area_names: List[str] = ['main', 'contrib', 'non-free']):
distribution_name: str = 'Debian',
suites: List[str] = ['stretch', 'buster', 'bullseye'],
components: List[str] = ['main', 'contrib', 'non-free']):
"""Initialize the debian data model.
Args:
db_engine: SQLAlchemy manipulation database object
override_conf: Override conf to pass to instantiate a lister
distributions: Default distribution to build
distribution_name: Distribution to initialize
suites: Default suites to register with the lister
components: Default components to register per suite
"""
distribution_name = 'Debian'
from swh.lister.debian.models import Distribution, Area
from sqlalchemy.orm import sessionmaker
db_session = sessionmaker(bind=db_engine)()
existing_distrib = db_session \
.query(Distribution) \
distrib = db_session.query(Distribution) \
.filter(Distribution.name == distribution_name) \
.one_or_none()
if not existing_distrib:
distrib = Distribution(name=distribution_name,
type='deb',
mirror_uri='http://deb.debian.org/debian/')
if distrib is None:
distrib = Distribution(
name=distribution_name, type='deb',
mirror_uri='http://deb.debian.org/debian/'
)
db_session.add(distrib)
for distribution_name in distributions:
for area_name in area_names:
area = Area(
name='%s/%s' % (distribution_name, area_name),
distribution=distrib,
)
db_session.add(area)
# Check the existing
existing_area = db_session.query(Area) \
.filter(Area.distribution == distrib) \
.all()
existing_area = set([a.name for a in existing_area])
logger.debug('Area already known: %s', ', '.join(existing_area))
# Create only the new ones
for suite in suites:
for component in components:
area_name = f'{suite}/{component}'
if area_name in existing_area:
logger.debug("Area '%s' already set, skipping", area_name)
continue
area = Area(name=area_name, distribution=distrib)
db_session.add(area)
db_session.commit()
db_session.commit()
db_session.close()
......
......@@ -19,9 +19,9 @@ def lister_debian(swh_listers):
lister = swh_listers['debian']
# Initialize the debian data model
debian_init(lister.db_engine,
distributions=['stretch'],
area_names=['main', 'contrib'])
debian_init(
lister.db_engine, suites=['stretch'], components=['main', 'contrib']
)
# Add the load-deb-package in the scheduler backend
lister.scheduler.create_task_type({
......
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import pytest
from swh.lister.debian import debian_init
from swh.lister.debian.models import Distribution, Area
@pytest.fixture
def engine(session):
session.autoflush = False
return session.bind
def test_debian_init_step(engine, session):
distribution_name = 'KaliLinux'
distrib = session.query(Distribution) \
.filter(Distribution.name == distribution_name) \
.one_or_none()
assert distrib is None
all_area = session.query(Area).all()
assert all_area == []
suites = ['wheezy', 'jessie']
components = ['main', 'contrib']
debian_init(engine, distribution_name=distribution_name,
suites=suites, components=components)
distrib = session.query(Distribution) \
.filter(Distribution.name == distribution_name) \
.one_or_none()
assert distrib is not None
assert distrib.name == distribution_name
assert distrib.type == 'deb'
assert distrib.mirror_uri == 'http://deb.debian.org/debian/'
all_area = session.query(Area).all()
assert len(all_area) == 2 * 2, "2 suites * 2 components per suite"
expected_area_names = []
for suite in suites:
for component in components:
expected_area_names.append(f'{suite}/{component}')
for area in all_area:
area.id = None
assert area.distribution == distrib
assert area.name in expected_area_names
# check idempotency (on exact same call)
debian_init(engine, distribution_name=distribution_name,
suites=suites, components=components)
distribs = session.query(Distribution) \
.filter(Distribution.name == distribution_name) \
.all()
assert len(distribs) == 1
distrib = distribs[0]
all_area = session.query(Area).all()
assert len(all_area) == 2 * 2, "2 suites * 2 components per suite"
# Add a new suite
debian_init(engine, distribution_name=distribution_name,
suites=['lenny'], components=components)
all_area = [a.name for a in session.query(Area).all()]
assert len(all_area) == (2 + 1) * 2, "3 suites * 2 components per suite"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment