--- # ============================================================================= # MetaVox Load Test - Hoofd Playbook # ============================================================================= # Maakt teamfolders, metadata-velden, bestanden en metadata-records aan op een # Nextcloud-instantie met MetaVox. # # Gebruikt directe filesystem writes en MySQL inserts i.p.v. WebDAV/API # voor maximale snelheid (~500K bestanden + ~43M metadata records in < 2 uur). # # Gebruik: ansible-playbook -i inventory/hosts.yml site.yml # Tags: precheck, teamfolders, metadata-fields, files, db-register, # folder-structure, drop-indexes, metadata-records # ============================================================================= - name: MetaVox Load Test - Setup en Data Generatie hosts: nextcloud_server gather_facts: true vars: nc_api_base: "{{ nextcloud_url }}/ocs/v2.php" nc_dav_base: "{{ nextcloud_url }}/remote.php/dav" nc_index_base: "{{ nextcloud_url }}/index.php" nc_auth_header: "Basic {{ (nextcloud_admin_user + ':' + nextcloud_admin_password) | b64encode }}" common_headers: OCS-APIRequest: "true" Accept: "application/json" Authorization: "{{ nc_auth_header }}" tasks: # ========================================================================= # FASE 0: Pre-checks # ========================================================================= - name: "Pre-check: Controleer of Nextcloud bereikbaar is" ansible.builtin.uri: url: "{{ nextcloud_url }}/status.php" method: GET return_content: true timeout: 30 register: nc_status failed_when: nc_status.status != 200 tags: [precheck] - name: "Pre-check: Toon Nextcloud versie" ansible.builtin.debug: msg: "Nextcloud is bereikbaar. Versie: {{ (nc_status.content | from_json).versionstring }}" tags: [precheck] - name: "Pre-check: Controleer beschikbare schijfruimte" ansible.builtin.command: df -h /var/www/nextcloud/data register: disk_check changed_when: false tags: [precheck] - name: "Pre-check: Toon schijfruimte" ansible.builtin.debug: msg: "{{ disk_check.stdout }}" tags: [precheck] # ========================================================================= # FASE 0.5: Maak groep aan (optioneel) # ========================================================================= - name: "Setup: Maak loadtest groep aan" ansible.builtin.uri: url: "{{ nc_api_base }}/cloud/groups" method: POST headers: "{{ common_headers }}" body_format: form-urlencoded body: groupid: "{{ loadtest_group }}" status_code: [200, 400] timeout: "{{ http_timeout }}" when: create_group | default(true) tags: [setup, teamfolders] # ========================================================================= # FASE 1: Teamfolders aanmaken via API # ========================================================================= - name: "Teamfolders: Genereer lijst van teamfolder namen" ansible.builtin.set_fact: teamfolder_names: "{{ teamfolder_names | default([]) + [teamfolder_prefix + '-%03d' | format(item)] }}" loop: "{{ range(1, num_teamfolders + 1) | list }}" tags: [teamfolders] - name: "Teamfolders: Maak {{ num_teamfolders }} teamfolders aan" ansible.builtin.uri: url: "{{ nc_index_base }}/apps/groupfolders/folders" method: POST headers: "{{ common_headers }}" body_format: form-urlencoded body: mountpoint: "{{ item }}" return_content: true status_code: [200] timeout: "{{ http_timeout }}" loop: "{{ teamfolder_names }}" register: teamfolder_results loop_control: label: "{{ item }}" pause: 0.2 tags: [teamfolders] - name: "Teamfolders: Verzamel folder IDs" ansible.builtin.set_fact: teamfolder_ids: "{{ teamfolder_results.results | map(attribute='json') | map(attribute='ocs') | map(attribute='data') | map(attribute='id') | list }}" tags: [teamfolders] when: teamfolder_results is defined - name: "Teamfolders: Ken groep '{{ loadtest_group }}' toe aan elke teamfolder" ansible.builtin.uri: url: "{{ nc_index_base }}/apps/groupfolders/folders/{{ item }}/groups" method: POST headers: "{{ common_headers }}" body_format: form-urlencoded body: group: "{{ loadtest_group }}" status_code: [200] timeout: "{{ http_timeout }}" loop: "{{ teamfolder_ids }}" loop_control: label: "Folder ID {{ item }}" pause: 0.1 tags: [teamfolders] - name: "Teamfolders: Stel permissies in voor de groep" ansible.builtin.uri: url: "{{ nc_index_base }}/apps/groupfolders/folders/{{ item }}/groups/{{ loadtest_group }}" method: POST headers: "{{ common_headers }}" body_format: form-urlencoded body: permissions: "{{ teamfolder_permissions }}" status_code: [200] timeout: "{{ http_timeout }}" loop: "{{ teamfolder_ids }}" loop_control: label: "Folder ID {{ item }}" pause: 0.1 tags: [teamfolders] - name: "Teamfolders: Resultaat" ansible.builtin.debug: msg: "{{ num_teamfolders }} teamfolders aangemaakt met IDs: {{ teamfolder_ids[:5] }}... (eerste 5 getoond)" tags: [teamfolders] # ========================================================================= # FASE 2: Metadata velden aanmaken (MetaVox OCS API) # ========================================================================= - name: "Metadata: Kopieer metadata-setup script" ansible.builtin.template: src: templates/setup_metadata_fields.py.j2 dest: /tmp/metavox_setup_metadata_fields.py mode: '0755' tags: [metadata-fields] - name: "Metadata: Maak metadata velddefinities aan via MetaVox API" ansible.builtin.command: cmd: python3 /tmp/metavox_setup_metadata_fields.py environment: PYTHONUNBUFFERED: "1" register: metadata_fields_result changed_when: "'created' in metadata_fields_result.stdout" tags: [metadata-fields] - name: "Metadata: Toon resultaat" ansible.builtin.debug: msg: "{{ metadata_fields_result.stdout_lines | default(['Geen output']) }}" tags: [metadata-fields] # ========================================================================= # FASE 3: Bestanden aanmaken (direct filesystem) # ========================================================================= - name: "Bestanden: Kopieer file-creatie script" ansible.builtin.template: src: templates/fast_create_files.sh.j2 dest: /tmp/metavox_fast_create_files.sh mode: '0755' tags: [files] - name: "Bestanden: Maak {{ files_per_teamfolder * num_teamfolders }} bestanden aan op filesystem" ansible.builtin.command: cmd: bash /tmp/metavox_fast_create_files.sh register: create_files_result changed_when: true async: 7200 poll: 30 tags: [files] - name: "Bestanden: Toon resultaat" ansible.builtin.debug: msg: "{{ create_files_result.stdout_lines[-10:] | default(['Geen output']) }}" tags: [files] # ========================================================================= # FASE 4: Bestanden registreren in database (direct MySQL) # ========================================================================= - name: "DB Register: Kopieer database registratie script" ansible.builtin.template: src: templates/fast_db_register.py.j2 dest: /tmp/metavox_fast_db_register.py mode: '0755' tags: [db-register] - name: "DB Register: Registreer bestanden in oc_filecache" ansible.builtin.command: cmd: python3 /tmp/metavox_fast_db_register.py environment: PYTHONUNBUFFERED: "1" register: db_register_result changed_when: true async: 7200 poll: 30 tags: [db-register] - name: "DB Register: Toon resultaat" ansible.builtin.debug: msg: "{{ db_register_result.stdout_lines[-10:] | default(['Geen output']) }}" tags: [db-register] # ========================================================================= # FASE 5: Mappenstructuur aanmaken en bestanden verplaatsen # ========================================================================= - name: "Mappen: Kopieer mappenstructuur script" ansible.builtin.template: src: templates/create_folder_structure.py.j2 dest: /tmp/metavox_create_folder_structure.py mode: '0755' tags: [folder-structure] - name: "Mappen: Maak mappenstructuur aan en verplaats bestanden" ansible.builtin.command: cmd: python3 /tmp/metavox_create_folder_structure.py environment: PYTHONUNBUFFERED: "1" register: folder_structure_result changed_when: true async: 7200 poll: 30 tags: [folder-structure] - name: "Mappen: Toon resultaat" ansible.builtin.debug: msg: "{{ folder_structure_result.stdout_lines[-10:] | default(['Geen output']) }}" tags: [folder-structure] # ========================================================================= # FASE 6: Drop overbodige indexes voor performance # ========================================================================= - name: "Indexes: Drop overbodige indexes op oc_metavox_file_gf_meta" ansible.builtin.command: cmd: > mysql -u root {{ nextcloud_db_name }} -e "DROP INDEX IF EXISTS {{ item }} ON oc_metavox_file_gf_meta;" loop: "{{ drop_indexes }}" loop_control: label: "DROP INDEX {{ item }}" ignore_errors: true tags: [drop-indexes] # ========================================================================= # FASE 7: Metadata records invoegen (direct MySQL) # ========================================================================= - name: "Metadata Records: Kopieer metadata insert script" ansible.builtin.template: src: templates/fast_metadata_insert.py.j2 dest: /tmp/metavox_fast_metadata_insert.py mode: '0755' tags: [metadata-records] - name: "Metadata Records: Voeg metadata records in via MySQL" ansible.builtin.command: cmd: python3 /tmp/metavox_fast_metadata_insert.py environment: PYTHONUNBUFFERED: "1" register: metadata_insert_result changed_when: true async: 86400 poll: 60 tags: [metadata-records] - name: "Metadata Records: Toon resultaat" ansible.builtin.debug: msg: "{{ metadata_insert_result.stdout_lines[-10:] | default(['Geen output']) }}" tags: [metadata-records] # ========================================================================= # FASE 8: Fix ownership # ========================================================================= - name: "Fix: Zet eigenaar op www-data voor groupfolders" ansible.builtin.file: path: "{{ groupfolders_dir }}" owner: www-data group: www-data recurse: true tags: [fix-ownership] # ========================================================================= # SAMENVATTING # ========================================================================= - name: "Samenvatting" ansible.builtin.debug: msg: | ============================================================ MetaVox Load Test - Voltooid! ============================================================ Teamfolders aangemaakt: {{ num_teamfolders }} Teamfolder metadata-velden: {{ teamfolder_metadata_fields | length }} File metadata-velden: {{ file_metadata_fields | length }} Totaal metadata-velden: {{ teamfolder_metadata_fields | length + file_metadata_fields | length }} Bestanden per teamfolder: {{ files_per_teamfolder }} Totaal bestanden: {{ files_per_teamfolder * num_teamfolders }} Mappenstructuur: 10 hoofdmappen x 3 submappen Metadata records: ~{{ files_per_teamfolder * num_teamfolders * (file_metadata_fields | length) }} Gedropte indexes: {{ drop_indexes | length }} ============================================================ Methode: Direct filesystem + MySQL (geen WebDAV/API) ============================================================ tags: [always]