Samkey/metavox-loadtest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a1eda430c81bf8bad974c2edce58548e1fd76ca2
metavox-loadtest/site.yml
Sam428-png a1eda430c8 Initial commit: MetaVox loadtest playbook 
Ansible playbook voor het opzetten van een MetaVox loadtest omgeving:
- 50 teamfolders met 10.000 bestanden elk (500K totaal)
- 100 metadata velddefinities (10 teamfolder + 90 file-level)
- 3-niveau mappenstructuur (10 hoofdmappen x 3 submappen)
- ~43M metadata records via directe MySQL inserts
- Geoptimaliseerde database indexes (7 redundante indexes gedropt)

Gebruikt directe filesystem writes en MySQL inserts i.p.v. WebDAV/API
voor maximale performance.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-16 16:55:30 +01:00

318 lines
12 KiB
YAML
Raw Blame History

---
# =============================================================================
# MetaVox Load Test - Hoofd Playbook
# =============================================================================
# Maakt teamfolders, metadata-velden, bestanden en metadata-records aan op een
# Nextcloud-instantie met MetaVox.
#
# Gebruikt directe filesystem writes en MySQL inserts i.p.v. WebDAV/API
# voor maximale snelheid (~500K bestanden + ~43M metadata records in < 2 uur).
#
# Gebruik: ansible-playbook -i inventory/hosts.yml site.yml
# Tags: precheck, teamfolders, metadata-fields, files, db-register,
# folder-structure, drop-indexes, metadata-records
# =============================================================================
- name: MetaVox Load Test - Setup en Data Generatie
hosts: nextcloud_server
gather_facts: true
vars:
nc_api_base: "{{ nextcloud_url }}/ocs/v2.php"
nc_dav_base: "{{ nextcloud_url }}/remote.php/dav"
nc_index_base: "{{ nextcloud_url }}/index.php"
nc_auth_header: "Basic {{ (nextcloud_admin_user + ':' + nextcloud_admin_password) | b64encode }}"
common_headers:
OCS-APIRequest: "true"
Accept: "application/json"
Authorization: "{{ nc_auth_header }}"
tasks:
# =========================================================================
# FASE 0: Pre-checks
# =========================================================================
- name: "Pre-check: Controleer of Nextcloud bereikbaar is"
ansible.builtin.uri:
url: "{{ nextcloud_url }}/status.php"
method: GET
return_content: true
timeout: 30
register: nc_status
failed_when: nc_status.status != 200
tags: [precheck]
- name: "Pre-check: Toon Nextcloud versie"
ansible.builtin.debug:
msg: "Nextcloud is bereikbaar. Versie: {{ (nc_status.content | from_json).versionstring }}"
tags: [precheck]
- name: "Pre-check: Controleer beschikbare schijfruimte"
ansible.builtin.command: df -h /var/www/nextcloud/data
register: disk_check
changed_when: false
tags: [precheck]
- name: "Pre-check: Toon schijfruimte"
ansible.builtin.debug:
msg: "{{ disk_check.stdout }}"
tags: [precheck]
# =========================================================================
# FASE 0.5: Maak groep aan (optioneel)
# =========================================================================
- name: "Setup: Maak loadtest groep aan"
ansible.builtin.uri:
url: "{{ nc_api_base }}/cloud/groups"
method: POST
headers: "{{ common_headers }}"
body_format: form-urlencoded
body:
groupid: "{{ loadtest_group }}"
status_code: [200, 400]
timeout: "{{ http_timeout }}"
when: create_group | default(true)
tags: [setup, teamfolders]
# =========================================================================
# FASE 1: Teamfolders aanmaken via API
# =========================================================================
- name: "Teamfolders: Genereer lijst van teamfolder namen"
ansible.builtin.set_fact:
teamfolder_names: "{{ teamfolder_names | default([]) + [teamfolder_prefix + '-%03d' | format(item)] }}"
loop: "{{ range(1, num_teamfolders + 1) | list }}"
tags: [teamfolders]
- name: "Teamfolders: Maak {{ num_teamfolders }} teamfolders aan"
ansible.builtin.uri:
url: "{{ nc_index_base }}/apps/groupfolders/folders"
method: POST
headers: "{{ common_headers }}"
body_format: form-urlencoded
body:
mountpoint: "{{ item }}"
return_content: true
status_code: [200]
timeout: "{{ http_timeout }}"
loop: "{{ teamfolder_names }}"
register: teamfolder_results
loop_control:
label: "{{ item }}"
pause: 0.2
tags: [teamfolders]
- name: "Teamfolders: Verzamel folder IDs"
ansible.builtin.set_fact:
teamfolder_ids: "{{ teamfolder_results.results | map(attribute='json') | map(attribute='ocs') | map(attribute='data') | map(attribute='id') | list }}"
tags: [teamfolders]
when: teamfolder_results is defined
- name: "Teamfolders: Ken groep '{{ loadtest_group }}' toe aan elke teamfolder"
ansible.builtin.uri:
url: "{{ nc_index_base }}/apps/groupfolders/folders/{{ item }}/groups"
method: POST
headers: "{{ common_headers }}"
body_format: form-urlencoded
body:
group: "{{ loadtest_group }}"
status_code: [200]
timeout: "{{ http_timeout }}"
loop: "{{ teamfolder_ids }}"
loop_control:
label: "Folder ID {{ item }}"
pause: 0.1
tags: [teamfolders]
- name: "Teamfolders: Stel permissies in voor de groep"
ansible.builtin.uri:
url: "{{ nc_index_base }}/apps/groupfolders/folders/{{ item }}/groups/{{ loadtest_group }}"
method: POST
headers: "{{ common_headers }}"
body_format: form-urlencoded
body:
permissions: "{{ teamfolder_permissions }}"
status_code: [200]
timeout: "{{ http_timeout }}"
loop: "{{ teamfolder_ids }}"
loop_control:
label: "Folder ID {{ item }}"
pause: 0.1
tags: [teamfolders]
- name: "Teamfolders: Resultaat"
ansible.builtin.debug:
msg: "{{ num_teamfolders }} teamfolders aangemaakt met IDs: {{ teamfolder_ids[:5] }}... (eerste 5 getoond)"
tags: [teamfolders]
# =========================================================================
# FASE 2: Metadata velden aanmaken (MetaVox OCS API)
# =========================================================================
- name: "Metadata: Kopieer metadata-setup script"
ansible.builtin.template:
src: templates/setup_metadata_fields.py.j2
dest: /tmp/metavox_setup_metadata_fields.py
mode: '0755'
tags: [metadata-fields]
- name: "Metadata: Maak metadata velddefinities aan via MetaVox API"
ansible.builtin.command:
cmd: python3 /tmp/metavox_setup_metadata_fields.py
environment:
PYTHONUNBUFFERED: "1"
register: metadata_fields_result
changed_when: "'created' in metadata_fields_result.stdout"
tags: [metadata-fields]
- name: "Metadata: Toon resultaat"
ansible.builtin.debug:
msg: "{{ metadata_fields_result.stdout_lines | default(['Geen output']) }}"
tags: [metadata-fields]
# =========================================================================
# FASE 3: Bestanden aanmaken (direct filesystem)
# =========================================================================
- name: "Bestanden: Kopieer file-creatie script"
ansible.builtin.template:
src: templates/fast_create_files.sh.j2
dest: /tmp/metavox_fast_create_files.sh
mode: '0755'
tags: [files]
- name: "Bestanden: Maak {{ files_per_teamfolder * num_teamfolders }} bestanden aan op filesystem"
ansible.builtin.command:
cmd: bash /tmp/metavox_fast_create_files.sh
register: create_files_result
changed_when: true
async: 7200
poll: 30
tags: [files]
- name: "Bestanden: Toon resultaat"
ansible.builtin.debug:
msg: "{{ create_files_result.stdout_lines[-10:] | default(['Geen output']) }}"
tags: [files]
# =========================================================================
# FASE 4: Bestanden registreren in database (direct MySQL)
# =========================================================================
- name: "DB Register: Kopieer database registratie script"
ansible.builtin.template:
src: templates/fast_db_register.py.j2
dest: /tmp/metavox_fast_db_register.py
mode: '0755'
tags: [db-register]
- name: "DB Register: Registreer bestanden in oc_filecache"
ansible.builtin.command:
cmd: python3 /tmp/metavox_fast_db_register.py
environment:
PYTHONUNBUFFERED: "1"
register: db_register_result
changed_when: true
async: 7200
poll: 30
tags: [db-register]
- name: "DB Register: Toon resultaat"
ansible.builtin.debug:
msg: "{{ db_register_result.stdout_lines[-10:] | default(['Geen output']) }}"
tags: [db-register]
# =========================================================================
# FASE 5: Mappenstructuur aanmaken en bestanden verplaatsen
# =========================================================================
- name: "Mappen: Kopieer mappenstructuur script"
ansible.builtin.template:
src: templates/create_folder_structure.py.j2
dest: /tmp/metavox_create_folder_structure.py
mode: '0755'
tags: [folder-structure]
- name: "Mappen: Maak mappenstructuur aan en verplaats bestanden"
ansible.builtin.command:
cmd: python3 /tmp/metavox_create_folder_structure.py
environment:
PYTHONUNBUFFERED: "1"
register: folder_structure_result
changed_when: true
async: 7200
poll: 30
tags: [folder-structure]
- name: "Mappen: Toon resultaat"
ansible.builtin.debug:
msg: "{{ folder_structure_result.stdout_lines[-10:] | default(['Geen output']) }}"
tags: [folder-structure]
# =========================================================================
# FASE 6: Drop overbodige indexes voor performance
# =========================================================================
- name: "Indexes: Drop overbodige indexes op oc_metavox_file_gf_meta"
ansible.builtin.command:
cmd: >
mysql -u root {{ nextcloud_db_name }} -e
"DROP INDEX IF EXISTS {{ item }} ON oc_metavox_file_gf_meta;"
loop: "{{ drop_indexes }}"
loop_control:
label: "DROP INDEX {{ item }}"
ignore_errors: true
tags: [drop-indexes]
# =========================================================================
# FASE 7: Metadata records invoegen (direct MySQL)
# =========================================================================
- name: "Metadata Records: Kopieer metadata insert script"
ansible.builtin.template:
src: templates/fast_metadata_insert.py.j2
dest: /tmp/metavox_fast_metadata_insert.py
mode: '0755'
tags: [metadata-records]
- name: "Metadata Records: Voeg metadata records in via MySQL"
ansible.builtin.command:
cmd: python3 /tmp/metavox_fast_metadata_insert.py
environment:
PYTHONUNBUFFERED: "1"
register: metadata_insert_result
changed_when: true
async: 86400
poll: 60
tags: [metadata-records]
- name: "Metadata Records: Toon resultaat"
ansible.builtin.debug:
msg: "{{ metadata_insert_result.stdout_lines[-10:] | default(['Geen output']) }}"
tags: [metadata-records]
# =========================================================================
# FASE 8: Fix ownership
# =========================================================================
- name: "Fix: Zet eigenaar op www-data voor groupfolders"
ansible.builtin.file:
path: "{{ groupfolders_dir }}"
owner: www-data
group: www-data
recurse: true
tags: [fix-ownership]
# =========================================================================
# SAMENVATTING
# =========================================================================
- name: "Samenvatting"
ansible.builtin.debug:
msg: |
============================================================
MetaVox Load Test - Voltooid!
============================================================
Teamfolders aangemaakt: {{ num_teamfolders }}
Teamfolder metadata-velden: {{ teamfolder_metadata_fields | length }}
File metadata-velden: {{ file_metadata_fields | length }}
Totaal metadata-velden: {{ teamfolder_metadata_fields | length + file_metadata_fields | length }}
Bestanden per teamfolder: {{ files_per_teamfolder }}
Totaal bestanden: {{ files_per_teamfolder * num_teamfolders }}
Mappenstructuur: 10 hoofdmappen x 3 submappen
Metadata records: ~{{ files_per_teamfolder * num_teamfolders * (file_metadata_fields | length) }}
Gedropte indexes: {{ drop_indexes | length }}
============================================================
Methode: Direct filesystem + MySQL (geen WebDAV/API)
============================================================
tags: [always]
Reference in New Issue View Git Blame Copy Permalink