Initial commit: MetaVox loadtest playbook
Ansible playbook voor het opzetten van een MetaVox loadtest omgeving: - 50 teamfolders met 10.000 bestanden elk (500K totaal) - 100 metadata velddefinities (10 teamfolder + 90 file-level) - 3-niveau mappenstructuur (10 hoofdmappen x 3 submappen) - ~43M metadata records via directe MySQL inserts - Geoptimaliseerde database indexes (7 redundante indexes gedropt) Gebruikt directe filesystem writes en MySQL inserts i.p.v. WebDAV/API voor maximale performance. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
317
site.yml
Normal file
317
site.yml
Normal file
@@ -0,0 +1,317 @@
|
||||
---
|
||||
# =============================================================================
|
||||
# MetaVox Load Test - Hoofd Playbook
|
||||
# =============================================================================
|
||||
# Maakt teamfolders, metadata-velden, bestanden en metadata-records aan op een
|
||||
# Nextcloud-instantie met MetaVox.
|
||||
#
|
||||
# Gebruikt directe filesystem writes en MySQL inserts i.p.v. WebDAV/API
|
||||
# voor maximale snelheid (~500K bestanden + ~43M metadata records in < 2 uur).
|
||||
#
|
||||
# Gebruik: ansible-playbook -i inventory/hosts.yml site.yml
|
||||
# Tags: precheck, teamfolders, metadata-fields, files, db-register,
|
||||
# folder-structure, drop-indexes, metadata-records
|
||||
# =============================================================================
|
||||
|
||||
- name: MetaVox Load Test - Setup en Data Generatie
|
||||
hosts: nextcloud_server
|
||||
gather_facts: true
|
||||
vars:
|
||||
nc_api_base: "{{ nextcloud_url }}/ocs/v2.php"
|
||||
nc_dav_base: "{{ nextcloud_url }}/remote.php/dav"
|
||||
nc_index_base: "{{ nextcloud_url }}/index.php"
|
||||
nc_auth_header: "Basic {{ (nextcloud_admin_user + ':' + nextcloud_admin_password) | b64encode }}"
|
||||
common_headers:
|
||||
OCS-APIRequest: "true"
|
||||
Accept: "application/json"
|
||||
Authorization: "{{ nc_auth_header }}"
|
||||
|
||||
tasks:
|
||||
# =========================================================================
|
||||
# FASE 0: Pre-checks
|
||||
# =========================================================================
|
||||
- name: "Pre-check: Controleer of Nextcloud bereikbaar is"
|
||||
ansible.builtin.uri:
|
||||
url: "{{ nextcloud_url }}/status.php"
|
||||
method: GET
|
||||
return_content: true
|
||||
timeout: 30
|
||||
register: nc_status
|
||||
failed_when: nc_status.status != 200
|
||||
tags: [precheck]
|
||||
|
||||
- name: "Pre-check: Toon Nextcloud versie"
|
||||
ansible.builtin.debug:
|
||||
msg: "Nextcloud is bereikbaar. Versie: {{ (nc_status.content | from_json).versionstring }}"
|
||||
tags: [precheck]
|
||||
|
||||
- name: "Pre-check: Controleer beschikbare schijfruimte"
|
||||
ansible.builtin.command: df -h /var/www/nextcloud/data
|
||||
register: disk_check
|
||||
changed_when: false
|
||||
tags: [precheck]
|
||||
|
||||
- name: "Pre-check: Toon schijfruimte"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ disk_check.stdout }}"
|
||||
tags: [precheck]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 0.5: Maak groep aan (optioneel)
|
||||
# =========================================================================
|
||||
- name: "Setup: Maak loadtest groep aan"
|
||||
ansible.builtin.uri:
|
||||
url: "{{ nc_api_base }}/cloud/groups"
|
||||
method: POST
|
||||
headers: "{{ common_headers }}"
|
||||
body_format: form-urlencoded
|
||||
body:
|
||||
groupid: "{{ loadtest_group }}"
|
||||
status_code: [200, 400]
|
||||
timeout: "{{ http_timeout }}"
|
||||
when: create_group | default(true)
|
||||
tags: [setup, teamfolders]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 1: Teamfolders aanmaken via API
|
||||
# =========================================================================
|
||||
- name: "Teamfolders: Genereer lijst van teamfolder namen"
|
||||
ansible.builtin.set_fact:
|
||||
teamfolder_names: "{{ teamfolder_names | default([]) + [teamfolder_prefix + '-%03d' | format(item)] }}"
|
||||
loop: "{{ range(1, num_teamfolders + 1) | list }}"
|
||||
tags: [teamfolders]
|
||||
|
||||
- name: "Teamfolders: Maak {{ num_teamfolders }} teamfolders aan"
|
||||
ansible.builtin.uri:
|
||||
url: "{{ nc_index_base }}/apps/groupfolders/folders"
|
||||
method: POST
|
||||
headers: "{{ common_headers }}"
|
||||
body_format: form-urlencoded
|
||||
body:
|
||||
mountpoint: "{{ item }}"
|
||||
return_content: true
|
||||
status_code: [200]
|
||||
timeout: "{{ http_timeout }}"
|
||||
loop: "{{ teamfolder_names }}"
|
||||
register: teamfolder_results
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
pause: 0.2
|
||||
tags: [teamfolders]
|
||||
|
||||
- name: "Teamfolders: Verzamel folder IDs"
|
||||
ansible.builtin.set_fact:
|
||||
teamfolder_ids: "{{ teamfolder_results.results | map(attribute='json') | map(attribute='ocs') | map(attribute='data') | map(attribute='id') | list }}"
|
||||
tags: [teamfolders]
|
||||
when: teamfolder_results is defined
|
||||
|
||||
- name: "Teamfolders: Ken groep '{{ loadtest_group }}' toe aan elke teamfolder"
|
||||
ansible.builtin.uri:
|
||||
url: "{{ nc_index_base }}/apps/groupfolders/folders/{{ item }}/groups"
|
||||
method: POST
|
||||
headers: "{{ common_headers }}"
|
||||
body_format: form-urlencoded
|
||||
body:
|
||||
group: "{{ loadtest_group }}"
|
||||
status_code: [200]
|
||||
timeout: "{{ http_timeout }}"
|
||||
loop: "{{ teamfolder_ids }}"
|
||||
loop_control:
|
||||
label: "Folder ID {{ item }}"
|
||||
pause: 0.1
|
||||
tags: [teamfolders]
|
||||
|
||||
- name: "Teamfolders: Stel permissies in voor de groep"
|
||||
ansible.builtin.uri:
|
||||
url: "{{ nc_index_base }}/apps/groupfolders/folders/{{ item }}/groups/{{ loadtest_group }}"
|
||||
method: POST
|
||||
headers: "{{ common_headers }}"
|
||||
body_format: form-urlencoded
|
||||
body:
|
||||
permissions: "{{ teamfolder_permissions }}"
|
||||
status_code: [200]
|
||||
timeout: "{{ http_timeout }}"
|
||||
loop: "{{ teamfolder_ids }}"
|
||||
loop_control:
|
||||
label: "Folder ID {{ item }}"
|
||||
pause: 0.1
|
||||
tags: [teamfolders]
|
||||
|
||||
- name: "Teamfolders: Resultaat"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ num_teamfolders }} teamfolders aangemaakt met IDs: {{ teamfolder_ids[:5] }}... (eerste 5 getoond)"
|
||||
tags: [teamfolders]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 2: Metadata velden aanmaken (MetaVox OCS API)
|
||||
# =========================================================================
|
||||
- name: "Metadata: Kopieer metadata-setup script"
|
||||
ansible.builtin.template:
|
||||
src: templates/setup_metadata_fields.py.j2
|
||||
dest: /tmp/metavox_setup_metadata_fields.py
|
||||
mode: '0755'
|
||||
tags: [metadata-fields]
|
||||
|
||||
- name: "Metadata: Maak metadata velddefinities aan via MetaVox API"
|
||||
ansible.builtin.command:
|
||||
cmd: python3 /tmp/metavox_setup_metadata_fields.py
|
||||
environment:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
register: metadata_fields_result
|
||||
changed_when: "'created' in metadata_fields_result.stdout"
|
||||
tags: [metadata-fields]
|
||||
|
||||
- name: "Metadata: Toon resultaat"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ metadata_fields_result.stdout_lines | default(['Geen output']) }}"
|
||||
tags: [metadata-fields]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 3: Bestanden aanmaken (direct filesystem)
|
||||
# =========================================================================
|
||||
- name: "Bestanden: Kopieer file-creatie script"
|
||||
ansible.builtin.template:
|
||||
src: templates/fast_create_files.sh.j2
|
||||
dest: /tmp/metavox_fast_create_files.sh
|
||||
mode: '0755'
|
||||
tags: [files]
|
||||
|
||||
- name: "Bestanden: Maak {{ files_per_teamfolder * num_teamfolders }} bestanden aan op filesystem"
|
||||
ansible.builtin.command:
|
||||
cmd: bash /tmp/metavox_fast_create_files.sh
|
||||
register: create_files_result
|
||||
changed_when: true
|
||||
async: 7200
|
||||
poll: 30
|
||||
tags: [files]
|
||||
|
||||
- name: "Bestanden: Toon resultaat"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ create_files_result.stdout_lines[-10:] | default(['Geen output']) }}"
|
||||
tags: [files]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 4: Bestanden registreren in database (direct MySQL)
|
||||
# =========================================================================
|
||||
- name: "DB Register: Kopieer database registratie script"
|
||||
ansible.builtin.template:
|
||||
src: templates/fast_db_register.py.j2
|
||||
dest: /tmp/metavox_fast_db_register.py
|
||||
mode: '0755'
|
||||
tags: [db-register]
|
||||
|
||||
- name: "DB Register: Registreer bestanden in oc_filecache"
|
||||
ansible.builtin.command:
|
||||
cmd: python3 /tmp/metavox_fast_db_register.py
|
||||
environment:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
register: db_register_result
|
||||
changed_when: true
|
||||
async: 7200
|
||||
poll: 30
|
||||
tags: [db-register]
|
||||
|
||||
- name: "DB Register: Toon resultaat"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ db_register_result.stdout_lines[-10:] | default(['Geen output']) }}"
|
||||
tags: [db-register]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 5: Mappenstructuur aanmaken en bestanden verplaatsen
|
||||
# =========================================================================
|
||||
- name: "Mappen: Kopieer mappenstructuur script"
|
||||
ansible.builtin.template:
|
||||
src: templates/create_folder_structure.py.j2
|
||||
dest: /tmp/metavox_create_folder_structure.py
|
||||
mode: '0755'
|
||||
tags: [folder-structure]
|
||||
|
||||
- name: "Mappen: Maak mappenstructuur aan en verplaats bestanden"
|
||||
ansible.builtin.command:
|
||||
cmd: python3 /tmp/metavox_create_folder_structure.py
|
||||
environment:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
register: folder_structure_result
|
||||
changed_when: true
|
||||
async: 7200
|
||||
poll: 30
|
||||
tags: [folder-structure]
|
||||
|
||||
- name: "Mappen: Toon resultaat"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ folder_structure_result.stdout_lines[-10:] | default(['Geen output']) }}"
|
||||
tags: [folder-structure]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 6: Drop overbodige indexes voor performance
|
||||
# =========================================================================
|
||||
- name: "Indexes: Drop overbodige indexes op oc_metavox_file_gf_meta"
|
||||
ansible.builtin.command:
|
||||
cmd: >
|
||||
mysql -u root {{ nextcloud_db_name }} -e
|
||||
"DROP INDEX IF EXISTS {{ item }} ON oc_metavox_file_gf_meta;"
|
||||
loop: "{{ drop_indexes }}"
|
||||
loop_control:
|
||||
label: "DROP INDEX {{ item }}"
|
||||
ignore_errors: true
|
||||
tags: [drop-indexes]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 7: Metadata records invoegen (direct MySQL)
|
||||
# =========================================================================
|
||||
- name: "Metadata Records: Kopieer metadata insert script"
|
||||
ansible.builtin.template:
|
||||
src: templates/fast_metadata_insert.py.j2
|
||||
dest: /tmp/metavox_fast_metadata_insert.py
|
||||
mode: '0755'
|
||||
tags: [metadata-records]
|
||||
|
||||
- name: "Metadata Records: Voeg metadata records in via MySQL"
|
||||
ansible.builtin.command:
|
||||
cmd: python3 /tmp/metavox_fast_metadata_insert.py
|
||||
environment:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
register: metadata_insert_result
|
||||
changed_when: true
|
||||
async: 86400
|
||||
poll: 60
|
||||
tags: [metadata-records]
|
||||
|
||||
- name: "Metadata Records: Toon resultaat"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ metadata_insert_result.stdout_lines[-10:] | default(['Geen output']) }}"
|
||||
tags: [metadata-records]
|
||||
|
||||
# =========================================================================
|
||||
# FASE 8: Fix ownership
|
||||
# =========================================================================
|
||||
- name: "Fix: Zet eigenaar op www-data voor groupfolders"
|
||||
ansible.builtin.file:
|
||||
path: "{{ groupfolders_dir }}"
|
||||
owner: www-data
|
||||
group: www-data
|
||||
recurse: true
|
||||
tags: [fix-ownership]
|
||||
|
||||
# =========================================================================
|
||||
# SAMENVATTING
|
||||
# =========================================================================
|
||||
- name: "Samenvatting"
|
||||
ansible.builtin.debug:
|
||||
msg: |
|
||||
============================================================
|
||||
MetaVox Load Test - Voltooid!
|
||||
============================================================
|
||||
Teamfolders aangemaakt: {{ num_teamfolders }}
|
||||
Teamfolder metadata-velden: {{ teamfolder_metadata_fields | length }}
|
||||
File metadata-velden: {{ file_metadata_fields | length }}
|
||||
Totaal metadata-velden: {{ teamfolder_metadata_fields | length + file_metadata_fields | length }}
|
||||
Bestanden per teamfolder: {{ files_per_teamfolder }}
|
||||
Totaal bestanden: {{ files_per_teamfolder * num_teamfolders }}
|
||||
Mappenstructuur: 10 hoofdmappen x 3 submappen
|
||||
Metadata records: ~{{ files_per_teamfolder * num_teamfolders * (file_metadata_fields | length) }}
|
||||
Gedropte indexes: {{ drop_indexes | length }}
|
||||
============================================================
|
||||
Methode: Direct filesystem + MySQL (geen WebDAV/API)
|
||||
============================================================
|
||||
tags: [always]
|
||||
Reference in New Issue
Block a user