Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
S
swh-core
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Platform
Development
swh-core
Commits
a057ad05
Commit
a057ad05
authored
6 years ago
by
David Douard
Browse files
Options
Downloads
Patches
Plain Diff
logger: kill the (now unused) PostresHandler
parent
19b1bece
No related branches found
No related tags found
1 merge request
!57
setup: extract db and http related parts in dedicated optional 'extras'
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
swh/core/logger.py
+0
-88
0 additions, 88 deletions
swh/core/logger.py
swh/core/tests/test_logger.py
+0
-51
0 additions, 51 deletions
swh/core/tests/test_logger.py
with
0 additions
and
139 deletions
swh/core/logger.py
+
0
−
88
View file @
a057ad05
...
...
@@ -5,11 +5,7 @@
import
datetime
import
logging
import
os
import
socket
import
psycopg2
from
psycopg2.extras
import
Json
from
systemd.journal
import
JournalHandler
as
_JournalHandler
,
send
try
:
...
...
@@ -80,90 +76,6 @@ def stringify(value):
return
str
(
value
)
class
PostgresHandler
(
logging
.
Handler
):
"""
log handler that store messages in a Postgres DB
See swh-core/swh/core/sql/log-schema.sql for the DB schema.
All logging methods can be used as usual. Additionally, arbitrary metadata
can be passed to logging methods, requesting that they will be stored in
the DB as a single JSONB value. To do so, pass a dictionary to the
'
extra
'
kwarg of any logging method; all keys in that dictionary that start with
EXTRA_LOGDATA_PREFIX (currently: ``swh_``) will be extracted to form the
JSONB dictionary. The prefix will be stripped and not included in the DB.
Note: the logger name will be used to fill the
'
module
'
DB column.
Sample usage::
logging.basicConfig(level=logging.INFO)
h = PostgresHandler(
'
dbname=softwareheritage-log
'
)
logging.getLogger().addHandler(h)
logger.info(
'
not so important notice
'
,
extra={
'
swh_type
'
:
'
swh_logging_test
'
,
'
swh_meditation
'
:
'
guru
'
})
logger.warn(
'
something weird just happened, did you see that?
'
)
"""
def
__init__
(
self
,
connstring
):
"""
Create a Postgres log handler.
Args:
config: configuration dictionary, with a key
"
log_db
"
containing a
libpq connection string to the log DB
"""
super
().
__init__
()
self
.
connstring
=
connstring
self
.
fqdn
=
socket
.
getfqdn
()
# cache FQDN value
def
_connect
(
self
):
return
psycopg2
.
connect
(
self
.
connstring
)
def
emit
(
self
,
record
):
msg
=
self
.
format
(
record
)
extra_data
=
get_extra_data
(
record
)
if
'
task
'
in
extra_data
:
task_args
=
{
'
args
'
:
extra_data
[
'
task
'
][
'
args
'
],
'
kwargs
'
:
extra_data
[
'
task
'
][
'
kwargs
'
],
}
try
:
json_args
=
Json
(
task_args
).
getquoted
()
except
TypeError
:
task_args
=
{
'
args
'
:
[
'
<failed to convert arguments to JSON>
'
],
'
kwargs
'
:
{},
}
else
:
json_args_length
=
len
(
json_args
)
if
json_args_length
>=
1000
:
task_args
=
{
'
args
'
:
[
'
<arguments too long>
'
],
'
kwargs
'
:
{},
}
extra_data
[
'
task
'
].
update
(
task_args
)
log_entry
=
(
db_level_of_py_level
(
record
.
levelno
),
msg
,
Json
(
extra_data
),
record
.
name
,
self
.
fqdn
,
os
.
getpid
())
db
=
self
.
_connect
()
with
db
.
cursor
()
as
cur
:
cur
.
execute
(
'
INSERT INTO log
'
'
(level, message, data, src_module, src_host, src_pid)
'
'
VALUES (%s, %s, %s, %s, %s, %s)
'
,
log_entry
)
db
.
commit
()
db
.
close
()
class
JournalHandler
(
_JournalHandler
):
def
emit
(
self
,
record
):
"""
Write `record` as a journal event.
...
...
This diff is collapsed.
Click to expand it.
swh/core/tests/test_logger.py
deleted
100644 → 0
+
0
−
51
View file @
19b1bece
# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import
logging
import
os
import
pytest
from
swh.core.logger
import
PostgresHandler
from
swh.core.tests
import
SQL_DIR
DUMP_FILE
=
os
.
path
.
join
(
SQL_DIR
,
'
log-schema.sql
'
)
@pytest.fixture
def
swh_db_logger
(
postgresql_proc
,
postgresql
):
cursor
=
postgresql
.
cursor
()
with
open
(
DUMP_FILE
)
as
fobj
:
cursor
.
execute
(
fobj
.
read
())
postgresql
.
commit
()
modname
=
'
swh.core.tests.test_logger
'
logger
=
logging
.
Logger
(
modname
,
logging
.
DEBUG
)
dsn
=
'
postgresql://{user}@{host}:{port}/{dbname}
'
.
format
(
host
=
postgresql_proc
.
host
,
port
=
postgresql_proc
.
port
,
user
=
'
postgres
'
,
dbname
=
'
tests
'
)
logger
.
addHandler
(
PostgresHandler
(
dsn
))
return
logger
@pytest.mark.db
def
test_log
(
swh_db_logger
,
postgresql
):
logger
=
swh_db_logger
modname
=
logger
.
name
logger
.
info
(
'
notice
'
,
extra
=
{
'
swh_type
'
:
'
test entry
'
,
'
swh_data
'
:
42
})
logger
.
warning
(
'
warning
'
)
with
postgresql
.
cursor
()
as
cur
:
cur
.
execute
(
'
SELECT level, message, data, src_module FROM log
'
)
db_log_entries
=
cur
.
fetchall
()
assert
(
'
info
'
,
'
notice
'
,
{
'
type
'
:
'
test entry
'
,
'
data
'
:
42
},
modname
)
in
db_log_entries
assert
(
'
warning
'
,
'
warning
'
,
{},
modname
)
in
db_log_entries
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment