From 18c9dad986a1f6f19d57dd97079dc22ad10b04df Mon Sep 17 00:00:00 2001
From: Nicolas Dandrimont <nicolas@dandrimont.eu>
Date: Mon, 7 May 2018 15:19:34 +0200
Subject: [PATCH] Only instantiate the storage backend once per import
This allows connection reuse for postgresql and potential remote backends such
as for the object storage, rather than reinitiating all connections on every
request.
---
swh/storage/api/server.py | 142 +++++++++++++++++++++-----------------
1 file changed, 78 insertions(+), 64 deletions(-)
diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py
index 23c48cc43..6be0d06e1 100644
--- a/swh/storage/api/server.py
+++ b/swh/storage/api/server.py
@@ -7,10 +7,10 @@ import json
import logging
import click
-from flask import g, request
+from flask import request
from swh.core import config
-from swh.storage import get_storage
+from swh.storage import get_storage as get_swhstorage
from swh.core.api import (SWHServerAPIApp, decode_request,
error_handler,
encode_data_server as encode_data)
@@ -34,6 +34,7 @@ DEFAULT_CONFIG = {
app = SWHServerAPIApp(__name__)
+storage = None
@app.errorhandler(Exception)
@@ -41,9 +42,12 @@ def my_error_handler(exception):
return error_handler(exception, encode_data)
-@app.before_request
-def before_request():
- g.storage = get_storage(**app.config['storage'])
+def get_storage():
+ global storage
+ if not storage:
+ storage = get_swhstorage(**app.config['storage'])
+
+ return storage
@app.route('/')
@@ -53,64 +57,66 @@ def index():
@app.route('/check_config', methods=['POST'])
def check_config():
- return encode_data(g.storage.check_config(**decode_request(request)))
+ return encode_data(get_storage().check_config(**decode_request(request)))
@app.route('/content/missing', methods=['POST'])
def content_missing():
- return encode_data(g.storage.content_missing(**decode_request(request)))
+ return encode_data(get_storage().content_missing(
+ **decode_request(request)))
@app.route('/content/missing/sha1', methods=['POST'])
def content_missing_per_sha1():
- return encode_data(g.storage.content_missing_per_sha1(
+ return encode_data(get_storage().content_missing_per_sha1(
**decode_request(request)))
@app.route('/content/present', methods=['POST'])
def content_find():
- return encode_data(g.storage.content_find(**decode_request(request)))
+ return encode_data(get_storage().content_find(**decode_request(request)))
@app.route('/content/add', methods=['POST'])
def content_add():
- return encode_data(g.storage.content_add(**decode_request(request)))
+ return encode_data(get_storage().content_add(**decode_request(request)))
@app.route('/content/update', methods=['POST'])
def content_update():
- return encode_data(g.storage.content_update(**decode_request(request)))
+ return encode_data(get_storage().content_update(**decode_request(request)))
@app.route('/content/data', methods=['POST'])
def content_get():
- return encode_data(g.storage.content_get(**decode_request(request)))
+ return encode_data(get_storage().content_get(**decode_request(request)))
@app.route('/content/metadata', methods=['POST'])
def content_get_metadata():
- return encode_data(g.storage.content_get_metadata(
+ return encode_data(get_storage().content_get_metadata(
**decode_request(request)))
@app.route('/directory', methods=['POST'])
def directory_get():
- return encode_data(g.storage.directory_get(**decode_request(request)))
+ return encode_data(get_storage().directory_get(**decode_request(request)))
@app.route('/directory/missing', methods=['POST'])
def directory_missing():
- return encode_data(g.storage.directory_missing(**decode_request(request)))
+ return encode_data(get_storage().directory_missing(
+ **decode_request(request)))
@app.route('/directory/add', methods=['POST'])
def directory_add():
- return encode_data(g.storage.directory_add(**decode_request(request)))
+ return encode_data(get_storage().directory_add(**decode_request(request)))
@app.route('/directory/path', methods=['POST'])
def directory_entry_get_by_path():
- return encode_data(g.storage.directory_entry_get_by_path(
+ return encode_data(get_storage().directory_entry_get_by_path(
**decode_request(request)))
@@ -118,249 +124,257 @@ def directory_entry_get_by_path():
def directory_ls():
dir = request.args['directory'].encode('utf-8', 'surrogateescape')
rec = json.loads(request.args.get('recursive', 'False').lower())
- return encode_data(g.storage.directory_ls(dir, recursive=rec))
+ return encode_data(get_storage().directory_ls(dir, recursive=rec))
@app.route('/revision/add', methods=['POST'])
def revision_add():
- return encode_data(g.storage.revision_add(**decode_request(request)))
+ return encode_data(get_storage().revision_add(**decode_request(request)))
@app.route('/revision', methods=['POST'])
def revision_get():
- return encode_data(g.storage.revision_get(**decode_request(request)))
+ return encode_data(get_storage().revision_get(**decode_request(request)))
@app.route('/revision/by', methods=['POST'])
def revision_get_by():
- return encode_data(g.storage.revision_get_by(**decode_request(request)))
+ return encode_data(get_storage().revision_get_by(
+ **decode_request(request)))
@app.route('/revision/log', methods=['POST'])
def revision_log():
- return encode_data(g.storage.revision_log(**decode_request(request)))
+ return encode_data(get_storage().revision_log(**decode_request(request)))
@app.route('/revision/logby', methods=['POST'])
def revision_log_by():
- return encode_data(g.storage.revision_log_by(**decode_request(request)))
+ return encode_data(get_storage().revision_log_by(
+ **decode_request(request)))
@app.route('/revision/shortlog', methods=['POST'])
def revision_shortlog():
- return encode_data(g.storage.revision_shortlog(**decode_request(request)))
+ return encode_data(get_storage().revision_shortlog(
+ **decode_request(request)))
@app.route('/revision/missing', methods=['POST'])
def revision_missing():
- return encode_data(g.storage.revision_missing(**decode_request(request)))
+ return encode_data(get_storage().revision_missing(
+ **decode_request(request)))
@app.route('/release/add', methods=['POST'])
def release_add():
- return encode_data(g.storage.release_add(**decode_request(request)))
+ return encode_data(get_storage().release_add(**decode_request(request)))
@app.route('/release', methods=['POST'])
def release_get():
- return encode_data(g.storage.release_get(**decode_request(request)))
+ return encode_data(get_storage().release_get(**decode_request(request)))
@app.route('/release/by', methods=['POST'])
def release_get_by():
- return encode_data(g.storage.release_get_by(**decode_request(request)))
+ return encode_data(get_storage().release_get_by(**decode_request(request)))
@app.route('/release/missing', methods=['POST'])
def release_missing():
- return encode_data(g.storage.release_missing(**decode_request(request)))
+ return encode_data(get_storage().release_missing(
+ **decode_request(request)))
@app.route('/object/find_by_sha1_git', methods=['POST'])
def object_find_by_sha1_git():
- return encode_data(g.storage.object_find_by_sha1_git(
+ return encode_data(get_storage().object_find_by_sha1_git(
**decode_request(request)))
@app.route('/occurrence', methods=['POST'])
def occurrence_get():
- return encode_data(g.storage.occurrence_get(**decode_request(request)))
+ return encode_data(get_storage().occurrence_get(**decode_request(request)))
@app.route('/occurrence/add', methods=['POST'])
def occurrence_add():
- return encode_data(g.storage.occurrence_add(**decode_request(request)))
+ return encode_data(get_storage().occurrence_add(**decode_request(request)))
@app.route('/snapshot/add', methods=['POST'])
def snapshot_add():
- return encode_data(g.storage.snapshot_add(**decode_request(request)))
+ return encode_data(get_storage().snapshot_add(**decode_request(request)))
@app.route('/snapshot', methods=['POST'])
def snapshot_get():
- return encode_data(g.storage.snapshot_get(**decode_request(request)))
+ return encode_data(get_storage().snapshot_get(**decode_request(request)))
@app.route('/snapshot/by_origin_visit', methods=['POST'])
def snapshot_get_by_origin_visit():
- return encode_data(g.storage.snapshot_get_by_origin_visit(
+ return encode_data(get_storage().snapshot_get_by_origin_visit(
**decode_request(request)))
@app.route('/snapshot/latest', methods=['POST'])
def snapshot_get_latest():
- return encode_data(g.storage.snapshot_get_latest(
+ return encode_data(get_storage().snapshot_get_latest(
**decode_request(request)))
@app.route('/origin/get', methods=['POST'])
def origin_get():
- return encode_data(g.storage.origin_get(**decode_request(request)))
+ return encode_data(get_storage().origin_get(**decode_request(request)))
@app.route('/origin/search', methods=['POST'])
def origin_search():
- return encode_data(g.storage.origin_search(**decode_request(request)))
+ return encode_data(get_storage().origin_search(**decode_request(request)))
@app.route('/origin/add_multi', methods=['POST'])
def origin_add():
- return encode_data(g.storage.origin_add(**decode_request(request)))
+ return encode_data(get_storage().origin_add(**decode_request(request)))
@app.route('/origin/add', methods=['POST'])
def origin_add_one():
- return encode_data(g.storage.origin_add_one(**decode_request(request)))
+ return encode_data(get_storage().origin_add_one(**decode_request(request)))
@app.route('/origin/visit/get', methods=['POST'])
def origin_visit_get():
- return encode_data(g.storage.origin_visit_get(**decode_request(request)))
+ return encode_data(get_storage().origin_visit_get(
+ **decode_request(request)))
@app.route('/origin/visit/getby', methods=['POST'])
def origin_visit_get_by():
return encode_data(
- g.storage.origin_visit_get_by(**decode_request(request)))
+ get_storage().origin_visit_get_by(**decode_request(request)))
@app.route('/origin/visit/add', methods=['POST'])
def origin_visit_add():
- return encode_data(g.storage.origin_visit_add(**decode_request(request)))
+ return encode_data(get_storage().origin_visit_add(
+ **decode_request(request)))
@app.route('/origin/visit/update', methods=['POST'])
def origin_visit_update():
- return encode_data(g.storage.origin_visit_update(
+ return encode_data(get_storage().origin_visit_update(
**decode_request(request)))
@app.route('/person', methods=['POST'])
def person_get():
- return encode_data(g.storage.person_get(**decode_request(request)))
+ return encode_data(get_storage().person_get(**decode_request(request)))
@app.route('/fetch_history', methods=['GET'])
def fetch_history_get():
- return encode_data(g.storage.fetch_history_get(request.args['id']))
+ return encode_data(get_storage().fetch_history_get(request.args['id']))
@app.route('/fetch_history/start', methods=['POST'])
def fetch_history_start():
return encode_data(
- g.storage.fetch_history_start(**decode_request(request)))
+ get_storage().fetch_history_start(**decode_request(request)))
@app.route('/fetch_history/end', methods=['POST'])
def fetch_history_end():
return encode_data(
- g.storage.fetch_history_end(**decode_request(request)))
+ get_storage().fetch_history_end(**decode_request(request)))
@app.route('/entity/add', methods=['POST'])
def entity_add():
return encode_data(
- g.storage.entity_add(**decode_request(request)))
+ get_storage().entity_add(**decode_request(request)))
@app.route('/entity/get', methods=['POST'])
def entity_get():
return encode_data(
- g.storage.entity_get(**decode_request(request)))
+ get_storage().entity_get(**decode_request(request)))
@app.route('/entity', methods=['GET'])
def entity_get_one():
- return encode_data(g.storage.entity_get_one(request.args['uuid']))
+ return encode_data(get_storage().entity_get_one(request.args['uuid']))
@app.route('/entity/from_lister_metadata', methods=['POST'])
def entity_from_lister_metadata():
- return encode_data(
- g.storage.entity_get_from_lister_metadata(**decode_request(request)))
+ return encode_data(get_storage().entity_get_from_lister_metadata(
+ **decode_request(request)))
@app.route('/tool/data', methods=['POST'])
def tool_get():
- return encode_data(g.storage.tool_get(
+ return encode_data(get_storage().tool_get(
**decode_request(request)))
@app.route('/tool/add', methods=['POST'])
def tool_add():
- return encode_data(g.storage.tool_add(
+ return encode_data(get_storage().tool_add(
**decode_request(request)))
@app.route('/origin/metadata/add', methods=['POST'])
def origin_metadata_add():
- return encode_data(g.storage.origin_metadata_add(**decode_request(
+ return encode_data(get_storage().origin_metadata_add(**decode_request(
request)))
@app.route('/origin/metadata/get', methods=['POST'])
def origin_metadata_get_by():
- return encode_data(g.storage.origin_metadata_get_by(**decode_request(
+ return encode_data(get_storage().origin_metadata_get_by(**decode_request(
request)))
@app.route('/provider/add', methods=['POST'])
def metadata_provider_add():
- return encode_data(g.storage.metadata_provider_add(**decode_request(
+ return encode_data(get_storage().metadata_provider_add(**decode_request(
request)))
@app.route('/provider/get', methods=['POST'])
def metadata_provider_get():
- return encode_data(g.storage.metadata_provider_get(**decode_request(
+ return encode_data(get_storage().metadata_provider_get(**decode_request(
request)))
@app.route('/provider/getby', methods=['POST'])
def metadata_provider_get_by():
- return encode_data(g.storage.metadata_provider_get_by(**decode_request(
+ return encode_data(get_storage().metadata_provider_get_by(**decode_request(
request)))
@app.route('/stat/counters', methods=['GET'])
def stat_counters():
- return encode_data(g.storage.stat_counters())
+ return encode_data(get_storage().stat_counters())
@app.route('/algos/diff_directories', methods=['POST'])
def diff_directories():
- return encode_data(g.storage.diff_directories(**decode_request(request)))
+ return encode_data(get_storage().diff_directories(
+ **decode_request(request)))
@app.route('/algos/diff_revisions', methods=['POST'])
def diff_revisions():
- return encode_data(g.storage.diff_revisions(**decode_request(request)))
+ return encode_data(get_storage().diff_revisions(**decode_request(request)))
@app.route('/algos/diff_revision', methods=['POST'])
def diff_revision():
- return encode_data(g.storage.diff_revision(**decode_request(request)))
+ return encode_data(get_storage().diff_revision(**decode_request(request)))
def run_from_webserver(environ, start_response,
--
GitLab