Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
S
swh-lister
Manage
Activity
Members
Labels
Plan
Issues
45
Issue boards
Milestones
Wiki
Code
Merge requests
6
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Platform
Development
swh-lister
Commits
5b652b30
Verified
Commit
5b652b30
authored
5 years ago
by
Antoine R. Dumont
Browse files
Options
Downloads
Patches
Plain Diff
lister.debian: Make debian init step idempotent and up-to-date
parent
4b9f0e05
No related branches found
Branches containing commit
Tags
v0.0.46
Tags containing commit
1 merge request
!132
lister.debian: Refactor and update the debian initialization step
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
swh/lister/debian/__init__.py
+36
-20
36 additions, 20 deletions
swh/lister/debian/__init__.py
swh/lister/debian/tests/conftest.py
+3
-3
3 additions, 3 deletions
swh/lister/debian/tests/conftest.py
swh/lister/debian/tests/test_init.py
+77
-0
77 additions, 0 deletions
swh/lister/debian/tests/test_init.py
with
116 additions
and
23 deletions
swh/lister/debian/__init__.py
+
36
−
20
View file @
5b652b30
...
...
@@ -3,46 +3,62 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import
logging
from
typing
import
Any
,
List
,
Mapping
logger
=
logging
.
getLogger
(
__name__
)
def
debian_init
(
db_engine
,
override_conf
:
Mapping
[
str
,
Any
]
=
{},
distributions
:
List
[
str
]
=
[
'
stretch
'
,
'
buster
'
],
area_names
:
List
[
str
]
=
[
'
main
'
,
'
contrib
'
,
'
non-free
'
]):
distribution_name
:
str
=
'
Debian
'
,
suites
:
List
[
str
]
=
[
'
stretch
'
,
'
buster
'
,
'
bullseye
'
],
components
:
List
[
str
]
=
[
'
main
'
,
'
contrib
'
,
'
non-free
'
]):
"""
Initialize the debian data model.
Args:
db_engine: SQLAlchemy manipulation database object
override_conf: Override conf to pass to instantiate a lister
distributions: Default distribution to build
distribution_name: Distribution to initialize
suites: Default suites to register with the lister
components: Default components to register per suite
"""
distribution_name
=
'
Debian
'
from
swh.lister.debian.models
import
Distribution
,
Area
from
sqlalchemy.orm
import
sessionmaker
db_session
=
sessionmaker
(
bind
=
db_engine
)()
existing_distrib
=
db_session
\
.
query
(
Distribution
)
\
distrib
=
db_session
.
query
(
Distribution
)
\
.
filter
(
Distribution
.
name
==
distribution_name
)
\
.
one_or_none
()
if
not
existing_distrib
:
distrib
=
Distribution
(
name
=
distribution_name
,
type
=
'
deb
'
,
mirror_uri
=
'
http://deb.debian.org/debian/
'
)
if
distrib
is
None
:
distrib
=
Distribution
(
name
=
distribution_name
,
type
=
'
deb
'
,
mirror_uri
=
'
http://deb.debian.org/debian/
'
)
db_session
.
add
(
distrib
)
for
distribution_name
in
distributions
:
for
area_name
in
area_names
:
area
=
Area
(
name
=
'
%s/%s
'
%
(
distribution_name
,
area_name
),
distribution
=
distrib
,
)
db_session
.
add
(
area
)
# Check the existing
existing_area
=
db_session
.
query
(
Area
)
\
.
filter
(
Area
.
distribution
==
distrib
)
\
.
all
()
existing_area
=
set
([
a
.
name
for
a
in
existing_area
])
logger
.
debug
(
'
Area already known: %s
'
,
'
,
'
.
join
(
existing_area
))
# Create only the new ones
for
suite
in
suites
:
for
component
in
components
:
area_name
=
f
'
{
suite
}
/
{
component
}
'
if
area_name
in
existing_area
:
logger
.
debug
(
"
Area
'
%s
'
already set, skipping
"
,
area_name
)
continue
area
=
Area
(
name
=
area_name
,
distribution
=
distrib
)
db_session
.
add
(
area
)
db_session
.
commit
()
db_session
.
commit
()
db_session
.
close
()
...
...
This diff is collapsed.
Click to expand it.
swh/lister/debian/tests/conftest.py
+
3
−
3
View file @
5b652b30
...
...
@@ -19,9 +19,9 @@ def lister_debian(swh_listers):
lister
=
swh_listers
[
'
debian
'
]
# Initialize the debian data model
debian_init
(
lister
.
db_engine
,
distributions
=
[
'
stretch
'
],
area_names
=
[
'
main
'
,
'
contrib
'
]
)
debian_init
(
lister
.
db_engine
,
suites
=
[
'
stretch
'
],
components
=
[
'
main
'
,
'
contrib
'
]
)
# Add the load-deb-package in the scheduler backend
lister
.
scheduler
.
create_task_type
({
...
...
This diff is collapsed.
Click to expand it.
swh/lister/debian/tests/test_init.py
0 → 100644
+
77
−
0
View file @
5b652b30
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import
pytest
from
swh.lister.debian
import
debian_init
from
swh.lister.debian.models
import
Distribution
,
Area
@pytest.fixture
def
engine
(
session
):
session
.
autoflush
=
False
return
session
.
bind
def
test_debian_init_step
(
engine
,
session
):
distribution_name
=
'
KaliLinux
'
distrib
=
session
.
query
(
Distribution
)
\
.
filter
(
Distribution
.
name
==
distribution_name
)
\
.
one_or_none
()
assert
distrib
is
None
all_area
=
session
.
query
(
Area
).
all
()
assert
all_area
==
[]
suites
=
[
'
wheezy
'
,
'
jessie
'
]
components
=
[
'
main
'
,
'
contrib
'
]
debian_init
(
engine
,
distribution_name
=
distribution_name
,
suites
=
suites
,
components
=
components
)
distrib
=
session
.
query
(
Distribution
)
\
.
filter
(
Distribution
.
name
==
distribution_name
)
\
.
one_or_none
()
assert
distrib
is
not
None
assert
distrib
.
name
==
distribution_name
assert
distrib
.
type
==
'
deb
'
assert
distrib
.
mirror_uri
==
'
http://deb.debian.org/debian/
'
all_area
=
session
.
query
(
Area
).
all
()
assert
len
(
all_area
)
==
2
*
2
,
"
2 suites * 2 components per suite
"
expected_area_names
=
[]
for
suite
in
suites
:
for
component
in
components
:
expected_area_names
.
append
(
f
'
{
suite
}
/
{
component
}
'
)
for
area
in
all_area
:
area
.
id
=
None
assert
area
.
distribution
==
distrib
assert
area
.
name
in
expected_area_names
# check idempotency (on exact same call)
debian_init
(
engine
,
distribution_name
=
distribution_name
,
suites
=
suites
,
components
=
components
)
distribs
=
session
.
query
(
Distribution
)
\
.
filter
(
Distribution
.
name
==
distribution_name
)
\
.
all
()
assert
len
(
distribs
)
==
1
distrib
=
distribs
[
0
]
all_area
=
session
.
query
(
Area
).
all
()
assert
len
(
all_area
)
==
2
*
2
,
"
2 suites * 2 components per suite
"
# Add a new suite
debian_init
(
engine
,
distribution_name
=
distribution_name
,
suites
=
[
'
lenny
'
],
components
=
components
)
all_area
=
[
a
.
name
for
a
in
session
.
query
(
Area
).
all
()]
assert
len
(
all_area
)
==
(
2
+
1
)
*
2
,
"
3 suites * 2 components per suite
"
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment