From 11b018899cffd405afd2391e203e863bc4ee77f9 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Tue, 23 Feb 2021 11:05:46 +0100
Subject: [PATCH] keycloak: Add deposit container to keycloak docker compose
So as to be able to play around with keycloak and deposit together.
Related to T2858
---
docker/conf/deposit-keycloak.yml | 30 +++
docker/docker-compose.keycloak.yml | 9 +
.../services/keycloak/keycloak_swh_setup.py | 235 ++++++++++--------
3 files changed, 171 insertions(+), 103 deletions(-)
create mode 100644 docker/conf/deposit-keycloak.yml
diff --git a/docker/conf/deposit-keycloak.yml b/docker/conf/deposit-keycloak.yml
new file mode 100644
index 0000000..88c1b4b
--- /dev/null
+++ b/docker/conf/deposit-keycloak.yml
@@ -0,0 +1,30 @@
+scheduler:
+ cls: remote
+ url: http://swh-scheduler:5008
+
+storage:
+ cls: remote
+ url: http://swh-storage:5002/
+
+storage_metadata:
+ cls: remote
+ url: http://swh-storage:5002/
+
+allowed_hosts:
+ - '*'
+
+private:
+ secret_key: prod-in-docker
+ db:
+ host: swh-deposit-db
+ port: 5432
+ name: swh-deposit
+ user: postgres
+ password: testpassword
+ media_root: /tmp/swh-deposit/uploads
+
+extraction_dir: "/tmp/swh-deposit/archive/"
+
+keycloak:
+ server_url: http://keycloak:8080/keycloak/auth/
+ realm_name: SoftwareHeritage
diff --git a/docker/docker-compose.keycloak.yml b/docker/docker-compose.keycloak.yml
index c43a62e..05b5f95 100644
--- a/docker/docker-compose.keycloak.yml
+++ b/docker/docker-compose.keycloak.yml
@@ -33,3 +33,12 @@ services:
SWH_CONFIG_FILENAME: /web-keycloak.yml
volumes:
- "./conf/web-keycloak.yml:/web-keycloak.yml:ro"
+
+ swh-deposit:
+ depends_on:
+ - keycloak
+ environment:
+ SWH_CONFIG_FILENAME: /deposit-keycloak.yml
+ volumes:
+ - "./conf/deposit-keycloak.yml:/deposit-keycloak.yml:ro"
+
diff --git a/docker/services/keycloak/keycloak_swh_setup.py b/docker/services/keycloak/keycloak_swh_setup.py
index f6a4d37..b0c0922 100755
--- a/docker/services/keycloak/keycloak_swh_setup.py
+++ b/docker/services/keycloak/keycloak_swh_setup.py
@@ -1,18 +1,24 @@
#!/usr/bin/env python3
-# Copyright (C) 2020 The Software Heritage developers
+# Copyright (C) 2020-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import logging
from keycloak import KeycloakAdmin
-server_url = "http://localhost:8080/keycloak/auth/"
-realm_name = "SoftwareHeritage"
-client_name = "swh-web"
+SERVER_URL = "http://localhost:8080/keycloak/auth/"
+REALM_NAME = "SoftwareHeritage"
-admin = {"username": "admin", "password": "admin"}
+CLIENT_WEBAPP_NAME = "swh-web"
+CLIENT_DEPOSIT_NAME = "swh-deposit"
+
+ADMIN = {"username": "admin", "password": "admin"}
+
+
+logger = logging.getLogger(__name__)
def assign_client_base_url(keycloak_admin, client_name, base_url):
@@ -36,38 +42,40 @@ def assign_client_roles_to_user(keycloak_admin, client_name, client_roles, usern
def create_user(keycloak_admin, user_data):
try:
keycloak_admin.create_user(user_data)
- except Exception:
- # user already created
- pass
+ except Exception as e:
+ logger.warning(f"User already created: {e}, skipping.")
def create_client_roles(keycloak_admin, client_name, client_roles):
for client_role in client_roles:
- keycloak_admin.create_client_role(client_name, payload={"name": client_role})
+ try:
+ keycloak_admin.create_client_role(client_name, payload={"name": client_role})
+ except Exception as e:
+ logger.warning(f"User already created: {e}, skipping.")
# login as admin in master realm
-keycloak_admin = KeycloakAdmin(server_url, admin["username"], admin["password"])
+KEYCLOAK_ADMIN = KeycloakAdmin(SERVER_URL, ADMIN["username"], ADMIN["password"])
# update master realm clients base urls as we use a reverse proxy
assign_client_base_url(
- keycloak_admin, "account", "/keycloak/auth/realms/master/account"
+ KEYCLOAK_ADMIN, "account", "/keycloak/auth/realms/master/account"
)
assign_client_base_url(
- keycloak_admin,
+ KEYCLOAK_ADMIN,
"security-admin-console",
"/keycloak/auth/admin/master/console/index.html",
)
-keycloak_admin.update_realm(
+KEYCLOAK_ADMIN.update_realm(
"master", payload={"loginTheme": "swh", "accountTheme": "swh", "adminTheme": "swh",}
)
# create swh realm
-keycloak_admin.create_realm(
+KEYCLOAK_ADMIN.create_realm(
payload={
- "realm": realm_name,
+ "realm": REALM_NAME,
"rememberMe": True,
"attributes": {"frontendUrl": "http://localhost:5080/keycloak/auth/"},
"enabled": True,
@@ -79,33 +87,33 @@ keycloak_admin.create_realm(
)
# set swh realm name in order to create users in it
-keycloak_admin.realm_name = realm_name
+KEYCLOAK_ADMIN.realm_name = REALM_NAME
# update swh realm clients base urls as we use a reverse proxy
assign_client_base_url(
- keycloak_admin, "account", f"/keycloak/auth/realms/{realm_name}/account"
+ KEYCLOAK_ADMIN, "account", f"/keycloak/auth/realms/{REALM_NAME}/account"
)
assign_client_base_url(
- keycloak_admin,
+ KEYCLOAK_ADMIN,
"security-admin-console",
- f"/keycloak/auth/admin/{realm_name}/console/index.html",
+ f"/keycloak/auth/admin/{REALM_NAME}/console/index.html",
)
# create an admin user in the swh realm
user_data = {
"email": "admin@example.org",
- "username": admin["username"],
- "firstName": admin["username"],
- "lastName": admin["username"],
+ "username": ADMIN["username"],
+ "firstName": ADMIN["username"],
+ "lastName": ADMIN["username"],
"credentials": [
- {"value": admin["username"], "type": admin["password"], "temporary": False}
+ {"value": ADMIN["username"], "type": ADMIN["password"], "temporary": False}
],
"enabled": True,
"emailVerified": False,
}
-create_user(keycloak_admin, user_data)
+create_user(KEYCLOAK_ADMIN, user_data)
# assign realm admin roles to created user
realm_management_roles = [
@@ -128,103 +136,124 @@ realm_management_roles = [
"manage-users",
"query-groups",
]
+
assign_client_roles_to_user(
- keycloak_admin, "realm-management", realm_management_roles, admin["username"]
+ KEYCLOAK_ADMIN, "realm-management", realm_management_roles, ADMIN["username"]
)
# login as admin in swh realm
-keycloak_admin = KeycloakAdmin(
- server_url, admin["username"], admin["password"], realm_name
+KEYCLOAK_ADMIN = KeycloakAdmin(
+ SERVER_URL, ADMIN["username"], ADMIN["password"], REALM_NAME
)
-# create swh-web public client
-keycloak_admin.create_client(
- payload={
- "id": client_name,
- "clientId": client_name,
- "surrogateAuthRequired": False,
- "enabled": True,
- "redirectUris": ["http://localhost:5004/*",],
- "bearerOnly": False,
- "consentRequired": False,
- "standardFlowEnabled": True,
- "implicitFlowEnabled": False,
- "directAccessGrantsEnabled": True,
- "serviceAccountsEnabled": False,
- "publicClient": True,
- "frontchannelLogout": False,
- "protocol": "openid-connect",
- "fullScopeAllowed": True,
- "protocolMappers": [
- {
- "name": "user groups",
- "protocol": "openid-connect",
- "protocolMapper": "oidc-group-membership-mapper",
- "consentRequired": False,
- "config": {
- "full.path": True,
- "userinfo.token.claim": True,
- "id.token.claim": True,
- "access.token.claim": True,
- "claim.name": "groups",
- "jsonType.label": "String",
+for (client_name, client_uri) in [
+ (CLIENT_WEBAPP_NAME, "http://localhost:5004/*"),
+ (CLIENT_DEPOSIT_NAME, "http://localhost:5006/*")
+]:
+ # create swh-web public client
+ KEYCLOAK_ADMIN.create_client(
+ payload={
+ "id": client_name,
+ "clientId": client_name,
+ "surrogateAuthRequired": False,
+ "enabled": True,
+ "redirectUris": [client_uri,],
+ "bearerOnly": False,
+ "consentRequired": False,
+ "standardFlowEnabled": True,
+ "implicitFlowEnabled": False,
+ "directAccessGrantsEnabled": True,
+ "serviceAccountsEnabled": False,
+ "publicClient": True,
+ "frontchannelLogout": False,
+ "protocol": "openid-connect",
+ "fullScopeAllowed": True,
+ "protocolMappers": [
+ {
+ "name": "user groups",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-group-membership-mapper",
+ "consentRequired": False,
+ "config": {
+ "full.path": True,
+ "userinfo.token.claim": True,
+ "id.token.claim": True,
+ "access.token.claim": True,
+ "claim.name": "groups",
+ "jsonType.label": "String",
+ },
},
- },
- {
- "name": "audience",
- "protocol": "openid-connect",
- "protocolMapper": "oidc-audience-mapper",
- "consentRequired": False,
- "config": {
- "included.client.audience": client_name,
- "id.token.claim": True,
- "access.token.claim": True,
+ {
+ "name": "audience",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-audience-mapper",
+ "consentRequired": False,
+ "config": {
+ "included.client.audience": client_name,
+ "id.token.claim": True,
+ "access.token.claim": True,
+ },
},
- },
- ],
- },
- skip_exists=True,
-)
+ ],
+ },
+ skip_exists=True,
+ )
# create staff group
-keycloak_admin.create_group(payload={"name": "staff",}, skip_exists=True)
+KEYCLOAK_ADMIN.create_group(payload={"name": "staff",}, skip_exists=True)
-groups = keycloak_admin.get_groups()
+GROUPS = KEYCLOAK_ADMIN.get_groups()
-admin_user_id = keycloak_admin.get_user_id(username=admin["username"])
+ADMIN_USER_ID = KEYCLOAK_ADMIN.get_user_id(username=ADMIN["username"])
-for group in groups:
- if group["name"] == "staff":
- keycloak_admin.group_user_add(admin_user_id, group["id"])
+for GROUP in GROUPS:
+ if GROUP["name"] == "staff":
+ KEYCLOAK_ADMIN.group_user_add(ADMIN_USER_ID, GROUP["id"])
break
-# create swh-web client roles
+# create webapp client roles
create_client_roles(
- keycloak_admin,
- client_name,
+ KEYCLOAK_ADMIN,
+ CLIENT_WEBAPP_NAME,
["swh.web.api.throttling_exempted", "swh.web.api.graph"],
)
-# create some test users
-user_data = {
- "email": "john.doe@example.org",
- "username": "johndoe",
- "firstName": "John",
- "lastName": "Doe",
- "credentials": [{"value": "johndoe-swh", "type": "password", "temporary": False}],
- "enabled": True,
- "emailVerified": False,
-}
-create_user(keycloak_admin, user_data)
+# create deposit client roles
+create_client_roles(
+ KEYCLOAK_ADMIN,
+ CLIENT_DEPOSIT_NAME,
+ ["swh.deposit.api"],
+)
-user_data = {
- "email": "jane.doe@example.org",
- "username": "janedoe",
- "firstName": "Jane",
- "lastName": "Doe",
- "credentials": [{"value": "janedoe-swh", "type": "password", "temporary": False}],
- "enabled": True,
- "emailVerified": False,
-}
-create_user(keycloak_admin, user_data)
+# create some test users
+for user_data in [
+ {
+ "email": "john.doe@example.org",
+ "username": "johndoe",
+ "firstName": "John",
+ "lastName": "Doe",
+ "credentials": [{"value": "johndoe-swh", "type": "password", "temporary": False}],
+ "enabled": True,
+ "emailVerified": False,
+ },
+ {
+ "email": "jane.doe@example.org",
+ "username": "janedoe",
+ "firstName": "Jane",
+ "lastName": "Doe",
+ "credentials": [{"value": "janedoe-swh", "type": "password", "temporary": False}],
+ "enabled": True,
+ "emailVerified": False,
+ },
+ {
+ "email": "",
+ "username": "hal",
+ "firstName": "HAL",
+ "lastName": "AI",
+ "credentials": [{"value": "test", "type": "password", "temporary": False}],
+ "enabled": True,
+ "emailVerified": False,
+ }
+]:
+ create_user(KEYCLOAK_ADMIN, user_data)
--
GitLab