diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e8d48a8f --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +#Git ignorance file for variant validator +#Testing script +VariantValidator/vvTest.py + +# docker config +configuration/docker.ini + +# Log +rest_api.log + +#pycs +rest_variantValidator/**/*.pyc +build +.idea +dist +rest_VariantValidator.egg-info +.DS_Store +__pycache__ diff --git a/Dockerfile b/Dockerfile index 236c921c..53dedadc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,40 @@ +FROM python:3.12.11 -FROM python:3.6 +# Set the working directory to /app +WORKDIR /app -#RUN seqrepo -r ${SEQREPO_DATA_DIR} pull -i ${SEQREPO_DATA_RELEASE} -#RUN touch ${SEQREPO_DATA_DIR}/testing.txt +# Copy the current directory contents into the container's /app directory +COPY . /app -#RUN apt update && apt install -y git +# Create logging directory +RUN mkdir /usr/local/share/logs -WORKDIR /app +# Update apt-get +RUN apt update -COPY . /app +# Install apt managed sofware +RUN apt -y install git \ + postgresql-client \ + sqlite3 \ + php -RUN apt-get update +# Manage git buffer +RUN git config http.postBuffer 500000000 +# Updrade pip RUN pip install --upgrade pip -RUN pip install -r REQUIREMENTS.txt - +# Install the tool RUN pip install -e . +# Copy the config file into the container home directory COPY configuration/docker.ini /root/.variantvalidator -CMD gunicorn -b 0.0.0.0:8000 app --threads=5 --worker-class=gthread --chdir ./rest_variantValidator/ \ No newline at end of file + +# Install LOVD Syntax checker +RUN python -m VariantValidator.bin.setup_lovd_syntax_checker + +# Define the entrypoint as an empty command +ENTRYPOINT [] + +# Start the container with CMD and set Gunicorn timeout to 600 seconds +CMD ["gunicorn", "-b", "0.0.0.0:8000", "--timeout", "600", "app", "--threads=5", "--chdir", "./rest_VariantValidator/"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..69a7fde6 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,104 @@ +pipeline { + agent { + docker { + image 'docker:24.0.6-git' + } + } + + environment { + CODECOV_TOKEN = credentials('CODECOV_TOKEN_rest_variantvalidator') + CONTAINER_SUFFIX = "${BUILD_NUMBER}" + DATA_VOLUME = "/root/variantvalidator_data/" + } + + stages { + stage("Clone Repository Remove dangling docker components and Create Docker Network") { + steps { + checkout scm + sh 'docker system prune --all --volumes --force' + } + } + stage("Switch to Git Branch") { + steps { + sh "git checkout ${BRANCH_NAME}" + sh "git pull" + } + } + stage("Install Docker Compose") { + steps { + sh 'apk update && apk add docker-compose' + } + } + stage("Check and Create Directories") { + steps { + script { + sh """ + if [ ! -d "${DATA_VOLUME}seqdata" ]; then + mkdir -p ${DATA_VOLUME}seqdata + fi + + if [ ! -d "${DATA_VOLUME}logs" ]; then + mkdir -p ${DATA_VOLUME}logs + fi + + ls -l ${DATA_VOLUME} + """ + } + } + } + stage("Build and Run containers") { + steps { + script { + sh """ + docker-compose --project-name rest-variantvalidator-ci build --no-cache rv-vvta rv-vdb rv-seqrepo rest-variantvalidator + docker-compose --project-name rest-variantvalidator-ci up -d rv-vvta && docker-compose --project-name rest-variantvalidator-ci up -d rv-vdb && docker-compose --project-name rest-variantvalidator-ci up -d rv-seqrepo && docker-compose --project-name rest-variantvalidator-ci up -d rest-variantvalidator + """ + } + } + } + stage("Connect and run Pytest") { + steps { + script { + def connectionSuccessful = false + for (int attempt = 1; attempt <= 5; attempt++) { + echo "Attempt $attempt to connect to the database..." + def exitCode = sh(script: ''' + docker-compose exec -e PGPASSWORD=uta_admin rest-variantvalidator-${CONTAINER_SUFFIX} psql -U uta_admin -d vvta -h rv-vvta-${CONTAINER_SUFFIX} -p 54321 + ''', returnStatus: true) + + if (exitCode == 0) { + connectionSuccessful = true + echo "Connected successfully! Running tests..." + break + } + + echo "Connection failed. Waiting for 60 seconds before the next attempt..." + sleep 60 + } + + if (!connectionSuccessful) { + error "All connection attempts failed. Exiting..." + } + } + } + } + stage("Run Pytest and Codecov") { + steps { + script { + sh 'docker-compose exec rest-variantvalidator-${CONTAINER_SUFFIX} pytest --cov=rest_VariantValidator --cov-report=term tests/' + sh 'docker-compose exec rest-variantvalidator-${CONTAINER_SUFFIX} codecov -t $CODECOV_TOKEN -b ${BRANCH_NAME}' + } + } + } + } + + post { + always { + script { + sh 'docker-compose down -v' + sh 'docker network rm $DOCKER_NETWORK' + sh 'docker system prune --all --volumes --force' + } + } + } +} diff --git a/README.md b/README.md index d62512ea..6ee45064 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ -# About rest_variantValidator +# rest_VariantValidator +[![Build Status](http://127.0.0.1:8080/job/rest_VariantValidator%20CI/job/master/badge/icon)](http://127.0.0.1:8080/job/rest_VariantValidator%20CI/job/master/) +[![codecov](https://codecov.io/gh/openvar/rest_variantValidator/graph/badge.svg?token=DE92ZVZT3F)](https://codecov.io/gh/openvar/rest_variantValidator) + +## About rest_variantValidator rest_variantValidator is a rest web interface for VariantValidator -# About VariantValidator +## About VariantValidator VariantValidator is a user-friendly software tool designed to validate the syntax and parameters of DNA variant descriptions according to the HGVS Sequence Variant @@ -47,14 +51,21 @@ Optional software: For installation instructions please see [INSTALLATION.md](./docs/INSTALLATION.md) -# Operation and configuration +## Operation and configuration Please see [MANUAL.md](./docs/MANUAL.md) +# Running in docker + +Please see [DOCKER.md](https://github.com/openvar/rest_variantValidator/blob/master/docs/DOCKER.md) + ## License Please see [LICENSE.txt](LICENSE.txt) +## Terms and conditions of use +[Terms and conditions can be found here](https://github.com/openvar/variantValidator/blob/master/README.md) + ## Cite us Hum Mutat. 2017 Oct 1. doi: 10.1002/humu.23348 diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt deleted file mode 100644 index 6fe8710c..00000000 --- a/REQUIREMENTS.txt +++ /dev/null @@ -1,21 +0,0 @@ -git+https://github.com/openvar/variantValidator@master#egg=VariantValidator -git+https://github.com/openvar/variantFormatter@master#egg=VariantFormatter -git+https://github.com/openvar/vv_flask-restful-swagger@master#egg=vv_flask_restful_swagger -flask -flask-log -flask-mail -flask-restful -gunicorn -biocommons.seqrepo>=0.5.1 -httplib2>=0.9.0 -configparser>=3.5.0 -pyliftover>=0.3 -biotools>=0.3.0 -mysql-connector-python -requests -pytest>=3.6 -pytest-cov -codecov - - - diff --git a/VERSION.txt b/VERSION.txt deleted file mode 100644 index 60453e69..00000000 --- a/VERSION.txt +++ /dev/null @@ -1 +0,0 @@ -v1.0.0 \ No newline at end of file diff --git a/batch/input.txt b/batch/input.txt new file mode 100644 index 00000000..e69de29b diff --git a/batch/output.txt b/batch/output.txt new file mode 100644 index 00000000..e69de29b diff --git a/bin/batch_validator.py b/bin/batch_validator.py new file mode 100644 index 00000000..d81387ac --- /dev/null +++ b/bin/batch_validator.py @@ -0,0 +1,101 @@ +import os +import sys +import VariantValidator +vval = VariantValidator.Validator() +cwd = os.path.dirname(os.path.abspath(__file__)) + + +# Check Args +if len(sys.argv) != 3: + print('Too few arguments. The command required is: python bin/batch_validator.py genome_build select_transcripts') + exit() + +# Check genome_build +genome_build = sys.argv[1] +genomes = ['GRCh38', 'hg38', 'GRCh37', 'hg19'] +if genome_build not in genomes: + warn = '%s is not a supported genome build' % genome_build + print(warn) + exit() + +select_transcripts = sys.argv[2] + +# Loop through file and detect fails +filepath = cwd.replace('bin', 'batch') +infile = filepath + '/input.txt' +outfile = open(filepath + "/output.txt", "w") +counter = 0 +with open(infile) as fp: + variants = fp.readlines() + for variant in variants: + variant = variant.strip() + print(variant) + try: + validate = vval.validate(variant, genome_build, select_transcripts) + validation = validate.format_as_table(with_meta=True) + if counter == 0: + line_counter = 0 + for line in validation: + if line_counter == 0: + outfile.write(line + '\n') + elif line_counter == 1: + ln_cat = '\t'.join(line) + outfile.write(ln_cat + '\n') + else: + copy_line = [] + for element in line: + if element == '': + copy_line.append('None') + elif element is None: + copy_line.append('None') + else: + copy_line.append(element) + ln_cat = '\t'.join(copy_line) + outfile.write(ln_cat + '\n') + line_counter = line_counter + 1 + else: + line_counter = 0 + for line in validation: + if line_counter == 0: + pass + elif line_counter == 1: + ln_cat = '\t'.join(line) + pass + else: + copy_line = [] + for element in line: + if element == '': + copy_line.append('None') + elif element is None: + copy_line.append('None') + else: + copy_line.append(element) + ln_cat = '\t'.join(copy_line) + outfile.write(ln_cat + '\n') + line_counter = line_counter + 1 + except VariantValidator.modules.utils.VariantValidatorError as e: + outfile.close() + print(variant) + print(e) + exit() + counter = counter + 1 + +print('Processing complete') +outfile.close() + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/bin/update_vdb.py b/bin/update_vdb.py new file mode 100644 index 00000000..a57f7495 --- /dev/null +++ b/bin/update_vdb.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python + +from VariantValidator import update_vv_db +import argparse + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--delete', '-d', action='store_true', help='Delete the contents of the current database ' + 'before updating') + + args = parser.parse_args() + if args.delete: + print("Deleting current database contents") + update_vv_db.delete() + + update_vv_db.update() + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/bin/variant_validator.py b/bin/variant_validator.py new file mode 100644 index 00000000..3847c04a --- /dev/null +++ b/bin/variant_validator.py @@ -0,0 +1,71 @@ +#! /usr/bin/env python + +import argparse +import sys +from VariantValidator import Validator + + +def output_results(valoutput, outformat, with_meta): + if outformat == 'dict': + return str(valoutput.format_as_dict(with_meta=with_meta)) + elif outformat == 'json': + return str(valoutput.format_as_json(with_meta=with_meta)) + else: + # table format + table = valoutput.format_as_table(with_meta=with_meta) + newtable = [] + for row in table: + if isinstance(row, list): + newrow = '\t'.join(row) + else: + newrow = str(row) + newtable.append(newrow) + return '\n'.join(newtable) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--variant', required=True, nargs='+', help="Variant(s) to validate") + parser.add_argument('-g', '--genome', nargs='?', default='GRCh37', choices=['GRCh37', 'GRCh38', 'hg19', 'hg38'], + help="Genome assembly (default: %(default)s)") + parser.add_argument('-t', '--transcripts', nargs='?', default='all', + help='Transcripts to output results for (default: %(default)s)') + parser.add_argument('-s', '--submission', choices=['individual', 'batch'], default='individual', + help='Submit variants individually or as a single batch validation (default: %(default)s)') + parser.add_argument('-f', '--output_format', choices=['dict', 'table', 'json'], default='dict', + help='Output validations as a list or as a dictionary (default: %(default)s)') + parser.add_argument('-o', '--output', type=argparse.FileType('w'), default='-', + help='Specifies the output file (default: stdout)') + parser.add_argument('-m', '--meta', action='store_true', default=False, + help='Also output metadata (default: %(default)s)') + + args = parser.parse_args() + + validator = Validator() + + if args.submission == 'individual': + for variant in args.variant: + output = validator.validate(variant, args.genome, args.transcripts) + args.output.write(output_results(output, args.output_format, args.meta) + '\n') + else: + batch = '|'.join(args.variant) + sys.stderr.write("Submitting batch query: %s\n" % batch) + output = validator.validate(batch, args.genome, args.transcripts) + args.output.write(output_results(output, args.output_format, args.meta) + '\n') + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/bin/vv_configure.py b/bin/vv_configure.py new file mode 100644 index 00000000..1d9d221c --- /dev/null +++ b/bin/vv_configure.py @@ -0,0 +1,76 @@ +#! /usr/bin/env python +from __future__ import print_function +import argparse +import os +import configparser +import pkgutil + + +def find_root(): + package = pkgutil.get_loader('VariantValidator') + path = os.path.dirname(os.path.dirname(package.get_filename())) + return path + + +def read_settings(): + root = find_root() + settings_file = os.path.join(root, 'VariantValidator', 'settings.py') + with open(settings_file) as f: + values = {} + exec(f.read(), {}, values) + return values + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--section', choices=['mysql', 'seqrepo', 'postgres', 'logging', 'EntrezID', 'liftover'], + nargs='?', help='Optional choice of section to configure') + + args = parser.parse_args() + + settings = read_settings() + newfile = False + + if os.path.exists(settings['CONFIG_DIR']): + readfile = settings['CONFIG_DIR'] + else: + root = find_root() + readfile = os.path.join(root, 'configuration', 'default.ini') + newfile = True + + config = configparser.ConfigParser() + config.read(readfile) + + values_changed = False + + for section in config.sections(): + if not newfile and args.section and args.section != section: + continue + print('Section:', section) + for name, value in config.items(section): + print("{} [{}]: ".format(name, value), end="") + newval = input() + if newval != '': + config.set(section, name, newval.strip()) + values_changed = True + + if newfile or values_changed: + with open(settings['CONFIG_DIR'], 'w') as fh: + config.write(fh) + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/configuration/.variantvalidator b/configuration/.variantvalidator new file mode 100644 index 00000000..f6028c20 --- /dev/null +++ b/configuration/.variantvalidator @@ -0,0 +1,29 @@ +[mysql] +host = 127.0.0.1 +port = 3306 +database = validator +user = USER +password = PASSWORD + +[seqrepo] +version = PATH TO SEQREPO VERSION +location = PATH TO SEQREPO +require_threading = True + +[postgres] +host = 127.0.0.1 +database = vvta +port = 5432 +version = VVTA VERSION +user = USERNAME +password = PASSWORD + +[logging] +log = True +console = INFO +file = WARNING + +[Entrez] +email = None +api_key = None + diff --git a/configuration/docker.ini b/configuration/docker.ini index 94b27318..cb7dea92 100644 --- a/configuration/docker.ini +++ b/configuration/docker.ini @@ -1,33 +1,37 @@ [mysql] -host = vdb +host = rv-vdb +port = 3306 database = validator user = vvadmin password = var1ant +version = vvdb_2025_3 [seqrepo] -version = 2018-08-21 -location = /usr/local/share/seqrepo +version = VV_SR_2025_02/master +location = /usr/local/share/seqdata +require_threading = True [postgres] -host = uta -database = uta -version = uta_20171026 -user = anonymous -password = +host = rv-vvta +port = 5432 +database = vvta +version = vvta_2025_02 +user = uta_admin +password = uta_admin [logging] -#Levels control verbosity and can be set to "CRITICAL" "ERROR" "WARNING" "INFO" or "DEBUG". +# Levels control verbosity and can be set to "CRITICAL" "ERROR" "WARNING" "INFO" or "DEBUG". log = True -console = ERROR +console = WARNING file = ERROR [Entrez] -email = -api_key = +email = OPTIONAL +api_key = OPTIONAL # -# Copyright (C) 2019 VariantValidator Contributors +# Copyright (C) 2016-2025 VariantValidator Contributors # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -41,4 +45,4 @@ api_key = # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -# \ No newline at end of file +# diff --git a/db_dockerfiles/seqrepo/Dockerfile b/db_dockerfiles/seqrepo/Dockerfile new file mode 100644 index 00000000..8d1811ff --- /dev/null +++ b/db_dockerfiles/seqrepo/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:22.04 + +RUN apt-get update + +RUN apt-get install -y wget + +RUN mkdir -p /usr/local/share/seqdata + +RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar https://www528.lamp.le.ac.uk/vvdata/vv_seqrepo/VV_SR_2025_02.tar + +RUN tar -xvf /usr/local/share/seqdata/VV_SR_2025_02.tar --directory /usr/local/share/seqdata + +RUN rm /usr/local/share/seqdata/VV_SR_2025_02.tar + +ENTRYPOINT [] + +CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/db_dockerfiles/vdb/Dockerfile b/db_dockerfiles/vdb/Dockerfile new file mode 100644 index 00000000..3f16ea19 --- /dev/null +++ b/db_dockerfiles/vdb/Dockerfile @@ -0,0 +1,20 @@ +# Should run on all processors +FROM ubuntu/mysql:8.0-22.04_beta + +ENV MYSQL_RANDOM_ROOT_PASSWORD yes +ENV MYSQL_DATABASE validator +ENV MYSQL_USER vvadmin +ENV MYSQL_PASSWORD var1ant + +RUN apt-get update && apt-get install -y \ + wget + +RUN rm -rf /var/lib/apt/lists/* + +# Set the max_connections directly in the my.cnf +RUN echo '[mysqld]' >> /etc/mysql/my.cnf && \ + echo 'max_connections=250' >> /etc/mysql/my.cnf + +RUN wget https://www528.lamp.le.ac.uk/vvdata/validator/validator_2025_03.sql.gz -O /docker-entrypoint-initdb.d/validator_2025_03.sql.gz + +CMD ["mysqld"] diff --git a/db_dockerfiles/vvta/Dockerfile b/db_dockerfiles/vvta/Dockerfile new file mode 100644 index 00000000..0706e8cb --- /dev/null +++ b/db_dockerfiles/vvta/Dockerfile @@ -0,0 +1,31 @@ +FROM postgres:14.9 + +ENV POSTGRES_DB=vvta +ENV POSTGRES_USER=uta_admin +ENV POSTGRES_PASSWORD=uta_admin + +# Install necessary dependencies +RUN apt-get update && \ + apt-get install -y \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Auto-create and set PostgreSQL configuration +RUN echo "shared_buffers = 2GB" > /docker-entrypoint-initdb.d/postgresql.conf + +# Step 1: Download the file +RUN wget https://www528.lamp.le.ac.uk/vvdata/vvta/vvta_2025_02_no_seq.sql.gz -O input_file.sql.gz + +# Step 2: Extract the gzipped file +RUN gzip -dq input_file.sql.gz + +# Step 3: Use sed to replace text +RUN sed 's/anyarray/anycompatiblearray/g' input_file.sql > modified_file.sql + +# Step 4: Compress the modified file +RUN rm input_file.sql +RUN gzip modified_file.sql + +# Step 5: Move the modified and compressed file to the desired location +RUN mv modified_file.sql.gz /docker-entrypoint-initdb.d/vvta_2025_02_no_seq.sql.gz + diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 00000000..14743001 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,46 @@ +version: '3' + +services: + dev-mode: + extends: + file: docker-compose.yml + service: rest-variantvalidator + depends_on: + - rv-vdb + - rv-vvta + - rv-seqrepo + volumes: + - rest_vv_dev_volume:/app + - vv-logs:/usr/local/share/logs # Mount volume for logs + - seqdata:/usr/local/share/seqdata # Mount volume for sequence data + ports: + - "5001:5000" + - "5050:5050" + - "8000:8000" + - "9000:9000" + expose: + - "5001" # Expose ports for external access + - "5050" + - "8000" + - "8080" + +volumes: + rest_vv_dev_volume: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${PWD}' + seqdata: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${HOME}/variantvalidator_data/seqdata' + vv-logs: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${HOME}/variantvalidator_data/logs' + diff --git a/docker-compose.yml b/docker-compose.yml index 24b73eb7..582917a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,65 @@ version: '3' services: - vdb: + rv-vdb: build: context: . - dockerfile: vdb_docker.df + dockerfile: db_dockerfiles/vdb/Dockerfile ports: - - "3306:3306" + - "33061:3306" expose: - - "3306" - uta: - image: biocommons/uta + - "33061" # Expose port for external access + + rv-vvta: + build: + context: . + dockerfile: db_dockerfiles/vvta/Dockerfile ports: - - "5432:5432" + - "54321:5432" expose: - - "5432" - seqrepo: - image: biocommons/seqrepo:2018-08-21 + - "54321" # Expose port for external access + shm_size: 2g + + rv-seqrepo: + build: + context: . + dockerfile: db_dockerfiles/seqrepo/Dockerfile volumes: - - seqdata:/usr/local/share/seqrepo - restvv: + - seqdata:/usr/local/share/seqdata # Mount volume for sequence data + + rest-variantvalidator: build: . + entrypoint: /bin/bash + command: ["-c", "sleep infinity"] + restart: always depends_on: - - vdb - - uta + - rv-vdb + - rv-vvta + - rv-seqrepo volumes: - - seqdata:/usr/local/share/seqrepo + - vv-logs:/usr/local/share/logs # Mount volume for logs + - seqdata:/usr/local/share/seqdata # Mount volume for sequence data ports: - - "5000:5000" + - "5001:5000" + - "5050:5050" - "8000:8000" + - "9000:9000" + expose: + - "5001" # Expose ports for external access + - "5050" + - "8000" + - "8080" volumes: - seqdata: \ No newline at end of file + seqdata: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${HOME}/variantvalidator_data/seqdata' + vv-logs: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${HOME}/variantvalidator_data/logs' \ No newline at end of file diff --git a/docs/.ipynb_checkpoints/rest_variantValidator_tutorial-checkpoint.ipynb b/docs/.ipynb_checkpoints/rest_variantValidator_tutorial-checkpoint.ipynb new file mode 100644 index 00000000..f3972ee1 --- /dev/null +++ b/docs/.ipynb_checkpoints/rest_variantValidator_tutorial-checkpoint.ipynb @@ -0,0 +1,586 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + "
Lecturer - Healthcare Sciences
\n", + "
(Clinical Bioinformatics)
\n", + "
The University of Manchester
\n", + " 		\n", + " \n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to SPRINT 1 notebook B\n", + "****\n", + "\n", + "## Overview\n", + "This introductory notebook, and subsequent SPRINT_introduction notebooks will continue to introduce the concept of retrieving data from Application Programming Interfaces [API](https://en.wikipedia.org/wiki/Application_programming_interface) which are web-hosted ([web API](https://en.wikipedia.org/wiki/Web_API))\n", + "\n", + "Many bioinformatics tools and data repositories can be accessed using web APIs including NCBI and Ensembl. \n", + "\n", + "Although we cannot hope to demonstrate how each an every useful bioinformatics web API works during this 10 week course, we will give you a broad overview of the tools we use to request data from these resources and the tools we use to make-sense of the data that are returned " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Table of Contents

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Notebook B, section 1: [Introduction to JSON](#json)\n", + "- [What is JSON](#watsit)\n", + "- [The JSON format](#jform)\n", + "- [Reading and writing JSON using Python](#jpy)\n", + "- [Section 1 Summary](#s1s)\n", + "\n", + "#### Notebook B, section 2: [Introduction to REST API](#restapi)\n", + "- [The REST framework](#rest)\n", + "- [Building a simple API: Part A - Build a simple REST API](#builder_a)\n", + "- [Building a simple API: Part B - Request data using Python](#builder_b)\n", + "- [Building a simple API: Part C - Create a new VariantValidator API](#builder_c)\n", + "- [Section 2 Summary and Assignment](#s2s)\n", + "- [Marked assessment](#practical)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
Learning Objective: Create functioning, standards compliant and well documented Python code
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "

Introduction to the VariantValidator REST API

\n", + " 		\n", + " \n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "Image by Peter Causey-Freeman" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

The REST framework

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [An Introduction to APIs](https://restful.io/an-introduction-to-api-s-cee90581ca1b) \n", + "- [Gonzalo Vázquez](https://restful.io/@gonzalovazquez)\n", + "- [Restful Web](https://restful.io/) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Using the VariantValidator REST API

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### API structure\n", + "\n", + "#### Framework\n", + "The VariantValidator REST API is built on the [Flask](https://en.wikipedia.org/wiki/Flask_\\(web_framework) web framework. \n", + "\n", + "The REST components are built using [Flask-RESTPlus](https://flask-restplus.readthedocs.io/en/stable/)\n", + "\n", + "> Flask-RESTPlus is an extension for Flask that adds support for quickly building REST APIs. Flask-RESTPlus encourages best practices with minimal setup. \n", + "\n", + ">It provides a coherent collection of tools to describe your API and expose its documentation properly (using Swagger).\n", + "\n", + "#### Namespaces and Endpoints\n", + "\n", + "The VariantValidator REST API has several tool-sets. Each set is divided into separate namespaces. \n", + "\n", + "For example, the namespace \"hello\" is used to test whether our services are up-and running. The namespaces and endpoints are most easily demonstrated by looking at the Swagger documented API on [https://rest.variantvalidator.org/](https://rest.variantvalidator.org/).\n", + "\n", + "The namespaces are\n", + "- VariantValidator; Core [VariantValidator](https://github.com/openvar/variantValidator) Python library\n", + "- VariantFormatter; [VariantFormatter](https://github.com/openvar/variantFormatter/tree/develop) extension library\n", + "- LOVD; Adapted endpoint for LOVD specific access to our resourced\n", + "- hello; Simple handshake allowing external users to test whether services are alive before submission\n", + "\n", + "Swagger documentation displays the namespaces as follows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![title](images/ns_and_ep.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each namespace contains endpoints which access specific functions of the VariantValidator libraries. For example the VariantValidator namespace has 3 endpoints\n", + "- gene2transcripts\n", + "- hgvs2reference\n", + "- variantvalidator\n", + "\n", + "### Building Queries\n", + "\n", + "In this interactive mode, the endpoint can be clicked allowing us to access a human-friendly query builder\n", + "\n", + "![title](images/query_builder.png)\n", + "\n", + "Currently the data can be returned in 2 different formats, JSON and XML. These are selected using the `Select the response format` drop-down menu. \n", + "\n", + "For this example I have selected the simple `gene2transcripts` endpoint which searches for all transcripts associated with a particular gene. The documentation tells us that me must input either a HGNC compliant gene symbol or a RefSeq transcript ID. However, this documentation will be improved because the tool also accepts RefSeq transcript IDs without version numbers, LRG IDs (*e.g.* LRG_1) and LRG transcript IDs (*e.g.* LRG_1t1). \n", + "\n", + "Once all the required fields are populated we can execute the query" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The API response\n", + "\n", + "Let's take a look at the response which Swagger has parsed into a user-friendly web page.\n", + "\n", + "![title](images/response.png)\n", + "\n", + "#### Server responses\n", + "1. [Response code](https://developer.amazon.com/docs/amazon-drive/ad-restful-api-response-codes.html) 200\n", + "2. The Response headers provide additional response metadata, *e.g.* the content-type and the time of the response\n", + "3. Response body, *i.e.* the JSON or XML the endpoint returns\n", + "\n", + "### The API query URLS\n", + "\n", + "Swagger also displays queries that can be used to trigger the response in a standard format, *i.e.* a non-interactive mode.\n", + "\n", + "![title](images/urls.png)\n", + "\n", + "#### curl\n", + "\n", + "curl is generally used in terminals and programming\n", + "\n", + "In this screen shot I have used a terminal to request data directly from the VariantValidator API using the provided curl. I have piped this into `python -m json.tool` to provide a pretty JSON display.\n", + "\n", + "The full request is `curl -X GET \"https://rest.variantvalidator.org/VariantValidator/tools/gene2transcripts/COL1A1?content-type=application%2Fjson\" -H \"accept: application/json\" | python -m json.tool`\n", + "\n", + "![title](images/curl.png)\n", + "\n", + "#### web URL\n", + "The web URL can simply be pasted into a browser and in the next section we will use the web URL to recover data from the VariantValidator API using Python\n", + "\n", + "![title](images/web.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Request data using Python

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The Requests module\n", + "[Requests](https://2.python-requests.org/en/master/)\n", + "> Requests: HTTP for Humans™\n", + "\n", + "> Requests is the only HTTP library for Python safe for human consumption\n", + "\n", + "***\n", + "Courtesy of the \"requests\" © 2019 Kenneth Reitz [Apache 2 License](https://www.apache.org/licenses/LICENSE-2.0)
\n", + "\n", + "OK, we have to take their word for it, but we are going to use requests because is's simple, easy to understand and is well maintained\n", + "\n", + "#### Method\n", + "\n", + "1. Install requests into your environment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install requests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "2. Import modules we will use" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "3. Create a simple function that calls the API using responses\n", + "\n", + " - *Note: This function is in a format that can be expanded*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "base_url = 'http://127.0.0.1:5000/'\n", + "def make_request(base_url, api_function):\n", + " # Tell the User the full URL of their call to the rest API\n", + " url = '%s%s' % (base_url, api_function)\n", + " print(\"Querying rest API with URL: \" + url)\n", + " \n", + " # Make the request and pass to a response object that the function returns\n", + " response = requests.get(url)\n", + " return response" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "4. Make a request to our API using the function. We need to specify the base_url and the api_function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = make_request(base_url, 'hello')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "5. Look at the response content\n", + " - response status code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response.status_code" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " - response headers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response.headers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "5. Finally, extract the body which the requests.json() method formats into Python dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "body = response.json()\n", + "body" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Building a simple API: Part C - Create a new VariantValidator API

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A bit basic isn't it?\n", + "\n", + "The simple `hello` API is a bit basic, but it does show you how an API works and we have also made requests to our API using Python.\n", + "\n", + "So what if we want to pass some data to the API?\n", + "\n", + "To `application/app_v2` I have added an additional **namespace** called name_space.\n", + "\n", + "I have also added a new API to our REST interface called name\n", + "\n", + "```python\n", + "name_space = application.namespace('name', description='Return a name provided by the user')\n", + "@name_space.route(\"\")\n", + "class NameClass(Resource):\n", + " def get(self, name):\n", + " return {\n", + " \"My name is\" : name\n", + " }\n", + "```\n", + "\n", + "To capture data submitted to the API, we tell the name_space.route to expect a **string** object\n", + "```python\n", + "@name_space.route(\"/\")\n", + "```\n", + "\n", + "and the NameClass Resource to expect the string object\n", + "```python\n", + "def get(self, name):\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "

Have a go

\n", + "\n", + "Activate `app_v2`\n", + "\n", + "```bash\n", + "$ python SPRINT/application/app_v2.py\n", + "```\n", + "\n", + "\n", + "### Exercise\n", + "\n", + "In a browser navigate to [http://127.0.0.1:5000/](http://127.0.0.1:5000/) and see whether you can figure out how to return your name using the API\n", + "\n", + "*Swagger is your friend here. It makes it very simple for a lay user to use an API*\n", + "\n", + "
\n", + "\n", + "### Exercise 2\n", + "\n", + "Now write a script that can make a call to the API and return the JSON that displays your name\n", + "\n", + "*Use the script above as a template. Remember, you may want to make a call to the hello API again, so keep the function flexible*\n", + "\n", + "Once the script is working, print out the response status, headings and JSON\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Section 2 Summary and Assignment

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Summary\n", + "In section 2 of this notebook we have learned about the REST API framework. We have learned how to build a simple REST API of our own. We have briefly touched upon the concept of how Swagger documentation makes APIs accessible to mere humans\\*. We have also learned how to request and make sense of data returned by REST PAIs using the Python requests module\n", + "\n", + "\\**We will look at of Swagger in more detail in week 8 of this unit* " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_style": "split" + }, + "source": [ + "\n", + "\n", + "### Over to you\n", + "\n", + "#### Aim of this exercise\n", + "The aim of this exercise is to keep you into the mindset of working together as a team. We will concentrate on aspects of working in an Agile fashion.\n", + "\n", + "#### Structure your team\n", + "Assign your team roles:\n", + "\n", + "1. **Project lead**\n", + " - Initiate the project on Git Issues (Note, there are two separate short projects here)\n", + " - Lead the group discussion in Git Issues and Slack\n", + " - Provide final feedback on the group's activities and close the issue\n", + " \n", + "\n", + "2. **Team members**\n", + " - Coders who will be responsible for writing the Python functions\n", + " - Testers who will be responsible for testing the code and providing feedback to the coders\n", + "\n", + "***We recommend ensuring that you most experienced coders work with your least experienced coders. Don't forget, this is a team assignment, if you can't figure out how to do something, ask your team on Slack!***" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_style": "split" + }, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_style": "split" + }, + "source": [ + "### Work-flow\n", + "\n", + "1. Group leader creates an issue on Git Issues\n", + "2. The coders will work together to write the module\n", + "3. The testers will review the final code and test the code. Feedback will be given to the coders within the Git issue\n", + "4. Once the coding is completed and tested, the project lead will summarise the key work-flow points and close the issue\n", + "\n", + "**Details about how the assignment will be marked can be found [here](LINK)**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "### Team Assignment\n", + "\n", + "***Remember, you are working as a team. Make sure you assign tasks in an agile way***\n", + "\n", + "#### Coding Workflow\n", + "1. In `applications/app_v3.py` I have created a `vv_space` namespace and `VariantValidatorClass` resource (Endpoint). Your task is to replace all the sections of the module marked \\_\\_\\_\\_\\_ (5 underscores) with actual code. The namespace requires 3 variables.\n", + "\n", + "When you have finished filling in the blanks, the answers can be found in `app_v4.py`\n", + "\n", + "
\n", + "\n", + "***Refer to the existing [VariantValidator REST API](https://rest.variantvalidator.org/webservices/variantvalidator.html#!/variantvalidator/VariantValidator)***\n", + "\n", + "2. In `applications/app_v3.py` create a new namespace and Endpoint that incorporates and returns the data from the function you created in SPRINT_1_introduction_a. When you are creating the namespace route, add a field that allows the user to select whether or not the sequences your function returns are displayed.\n", + " \n", + "*Note: for non-coding transcripts some of these fields will need to return None*\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Concluding remarks\n", + "We will cover methods for reading and writing JSON data to-and-from files in week_6, but a key aspect of learning to program is learning to use the internet to find out how to do things. Google and stack overflow are your friends!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 3b2d8d83..582254ca 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -1,97 +1,329 @@ -# Docker +# Running rest_VariantValidator in Docker -To install rest_variantValidator via Docker, first ensure you have both docker and docker-compose installed. +## Prerequisites + +To install rest_variantValidator via Docker, first ensure that you have both docker and docker-compose installed. See their [documentation](https://docs.docker.com/compose/install/) for information. -Then, clone the repository and move into that directory. + +## Clone the rest_VariantValidator Repository +Create a directory to collate your cloned repositories. Move into the directory, then clone the repository. ```bash $ git clone https://github.com/openvar/rest_variantValidator -cd rest_variantValidator/ +``` + +Once the repository has been cloned, cd into the rest_variantValidator directory that the clone creates. + +```bash +$ cd rest_variantValidator ``` -## Configure -Edit the file configuration/docker.ini -You will need to provide your email address and we recommend generating and using an [Entrez API key](https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/) +If you have cloned the repository previously, update it prior to installing/re-installing using Docker + +```bash +$ git pull +``` + +## Configuring the software + +Edit the file located in `configuration/docker.ini` +You will need to provide an email address + +**Optional (from VariantValidator v2.0.0 - September 2021)** Generate an Entrez API key. This will be necessary if you +do not update your container for more than 12 months; else leave as `None`. See +[Entrez API key](https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/) for details -*Note: configuration can be updated (see below for details)* +Note: Reconfiguration can be achieved by accessing the docker container through bash. See below for entry and the +VariantValidator [manual](https://github.com/openvar/variantValidator/blob/master/docs/MANUAL.md) for details +## Build the container -## Launch -You can then launch the docker containers and run them using +*Note: some of these steps take ~1hr to complete depending on the speed of your internet connection, particularly +compiling SeqRepo* +#### Build and startup procedure + +rest_VariantValidator can be built in Production mode and Development mode. Development mode mounts the root directory +of the host git Repository to the equivalent project directory in the docker container. This means that changes to the +code on the host machine are mapped into the container allowing on-the-fly development. It also allows you to view logs. +Choose one of the following commands to build and start the rest_VariantValidaor containers + +- Create directories for sharing resources between your computer and the containers +```bash +$ mkdir -p ~/variantvalidator_data/seqdata && mkdir -p ~/variantvalidator_data/logs + +- Production build +```bash +# Build +$ docker-compose build --no-cache rv-vvta rv-vdb rv-seqrepo rest-variantvalidator +``` +- Development and testing build ```bash -docker-compose up +# Build +$ docker-compose -f docker-compose.yml -f docker-compose-dev.yml build --no-cache rv-vvta rv-vdb rv-seqrepo dev-mode +``` +- The build stage has completed when you see the following message or something similar +``` + => [rest-variantvalidator 10/10] COPY configuration/docker.ini /root/.variantvalidator 0.0s + => [rest-variantvalidator] exporting to image 2.3s + => => exporting layers 2.3s + => => writing image sha256:097829685d99c7b308563dbee52009bc3dd7d79e85e195d454f3bf602afd5d95 0.0s + => => naming to docker.io/library/rest_variantvalidator-rest-variantvalidator ``` -Note, the first time this is run it will download each of the databases including the pre-populated -validator database and could take up to 30 minutes depending on your connection. We do not recommend -running this in the background as you need to see the logs and therefore when the databases are -ready to be used. +- Use this command to complete the standard build and wait for the above messages +```bash +$ docker-compose up -d rv-vvta && \ + docker-compose up -d rv-vdb && \ + docker-compose up -d rv-seqrepo && \ + docker-compose up -d rest-variantvalidator +``` +- Or for a development and testing build, swap for these commands -## Access rest_variantValidator -In a web browser navigate to -[0.0.0.0:8000](http://0.0.0.0:8000/) +```bash +$ docker-compose up -d rv-vvta && \ + docker-compose up -d rv-vdb && \ + docker-compose up -d rv-seqrepo && \ + docker-compose -f docker-compose.yml -f docker-compose-dev.yml up -d dev-mode +``` + +- You may need to run the above command a second time, until you see a response similar to +```bash + ✔ Container rest_variantvalidator-rv-vvta-1 Running 0.0s + ✔ Container rest_variantvalidator-rv-seqrepo-1 Running 0.0s + ✔ Container rest_variantvalidator-rv-vdb-1 Running 0.0s + ✔ Container rest_variantvalidator-dev-mode-1 Running +``` -## Stop the app -`ctrl+c` +- ***The first time you see all services running, wait for 30 minutes before testing and using the API. This is because the +VVTA database needs to load and initialise. There are no logs generated while this is happening*** -## Stop the remove the containers +### Test the build ```bash -$ docker-compose down +# Run PyTest (all tests should pass except for RunTimeError fails. These are intended for local development not dockerised versions) +$ docker exec rest_variantvalidator-rest-variantvalidator-1 pytest +``` +Note: Different host Operating Systems name the container using slightly different conventions e.g. underscores instead +of hyphens. To find your container name run the command + +```bash +$ docker ps +******************************************************************************************************************************************************************************************************************************************************************************* +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +75078f429d72 rest_variantvalidator-rest-variantvalidator "/bin/bash -c 'sleep…" 41 seconds ago Up 39 seconds 0.0.0.0:5000->5000/tcp, 0.0.0.0:5050->5050/tcp, 0.0.0.0:8000->8000/tcp, 0.0.0.0:9000->9000/tcp, 8080/tcp rest_variantvalidator-rest-variantvalidator-1 +******************************************************************************************************************************************************************************************************************************************************************************* ``` +***Note: In Development and testing builds, the container name will be e.g. rest_variantvalidator-dev-mode-1*** -## Run -You can go into the container via bash to use -[VariantValidator](https://github.com/openvar/variantValidator/blob/develop_v1/docs/MANUAL.md) directly. +# Run the server +```bash +# Start the container in detached mode +$ docker exec -it rest_variantvalidator-rest-variantvalidator-1 gunicorn -b 0.0.0.0:8000 --timeout 600 wsgi:app --threads=5 --chdir ./rest_VariantValidator/ +``` +Optional: If your docker instance has multiple available cores, you can increase processing power by starting multiple workers e.g. ```bash -$ docker-compose run restvv bash +docker exec -it rest_variantvalidator-rest-variantvalidator-1 gunicorn -b 0.0.0.0:8000 --workers 3 --timeout 600 wsgi:app --threads=5 --chdir ./rest_VariantValidator/ +``` + +***Note: In Development and testing builds, the container name will be e.g. rest_variantvalidator-dev-mode-1*** + +In a web browser navigate to +[http://0.0.0.0:8000](http://0.0.0.0:8000) + +When you are finished, stop the container ``` +ctrl + c +``` + +***Note: you may need to change :8080 to one of :5000 or :8000 depending on whether you altered the default port above*** + +### Build errors you may encounter + +***If you have MySQL and or Postgres databases already running, you may encounter an error*** + +> "ERROR: for vdb Cannot start service vdb: Ports are not available: listen tcp 0.0.0.0:3306: bind: address already in use" + +If you encounter these issues, stop the build by pressing `ctrl+c` + +- Reconfigure the ports used in the `docker-comose.yml` file as shown here +```yml +services: + vdb: + build: + context: . + dockerfile: vdb_docker.df + ports: + # - "33060:3306" + - "3306:3306" + expose: + # - "33060" + - "3306" + uta: + build: + context: . + dockerfile: uta_docker.df + ports: + - "54320:5432" + expose: + - "54320" + +``` +- hash (`#`) the conflicting port and add the new ports as shown above -Note, that each time one of these commands is run a new container is created. -For more information on how to use docker-compose see their [documentation](https://docs.docker.com/compose/). +Once complete, re-build as above +***You may encounter a build error relating to other unavailable ports*** + +> "Cannot start service restvv: Ports are not available: listen tcp 0.0.0.0:8000: bind: address already in use" + +If you encounter these issues, stop the build by pressing `ctrl+c` + +- Reconfigure the ports used in the `docker-comose.yml` file as shown here + +```yml + restvv: + build: . + depends_on: + - vdb + - uta + volumes: + - seqdata:/usr/local/share/seqrepo + ports: + - "5000:5000" + # - "8000:8000" + - "8080:8080" + expose: + - "5000" + # - "8000" + - 8080 +``` + +- hash (`#`) the conflicting port and add the new ports as shown above +- Change the command in Dockerfile to reflect the changes e.g. `CMD gunicorn -b 0.0.0.0:8080 app --threads=5 --chdir ./rest_VariantValidator/` + +Once complete, re-build as above + +## Accessing the VariantValidator databases externally It is possible to access both the UTA and Validator databases outside of docker as they expose the - default PostgreSQL and MySQL ports (5432 and 3306 respectively). In the current set-up it is not possible to - access the seqrepo database outside of docker. - -Finally, it should be noted that the current UTA docker container is not up-to-date and only contains the -2017-10-26 release. Therefore use caution when interpreting these results, and be advised the - VariantValidator tests will fail. + default PostgreSQL and MySQL ports (5432 and 3306 respectively). You can also access the seqrepo database outside of +docker. Navigate to ~/variantvalidator_data/seqdata -## Accessing VariantValidator and reconfiguring this container +## Accessing VariantValidator directly through bash and reconfiguring a container post build The container hosts a full install of VariantValidator. -VariantValidator can be run on the commandline from within the container. +To start this version you start the container in detached mode and access it using + +```bash +$ docker-compose exec rest_variantvalidator-rest-variantvalidator-1 bash +``` + +When you are finished exit the container + +```bash +$ exit +``` -Instructions can be found in the VariantValidator [manual](https://github.com/openvar/variantValidator/blob/develop_v1/docs/MANUAL.md) +#### Running the VariantValidator shell script -## Check which docker containers are running +Once installed and running it is possible to run VariantValidator via a bash shell using the running the container +**Example** ```bash -$ docker ps -a +# Note: The variant description must be contained in '' or "". See MANUAL.md for more examples +$ docker exec -it rest_variantvalidator-rest-variantvalidator-1 python bin/variant_validator.py -v 'NC_000017.11:g.50198002C>A' -g GRCh38 -t mane -s individual -f json -m ``` -## List all docker containers +## Developing VariantValidator in Docker +Create the development and testing build and changes you make in the cloned Repo should map into the container + +Create a new branch for your developments + ```bash -$ docker container ls -a +$ git branch name_of_branch +$ git checkout name_of_branch ``` -## Stop containers +You can then use the containers Python interpreter to run queries, e.g. + +```python +import json +import VariantValidator +vval = VariantValidator.Validator() +variant = 'NM_000088.3:c.589G>T' +genome_build = 'GRCh38' +select_transcripts = 'all' +validate = vval.validate(variant, genome_build, select_transcripts) +validation = validate.format_as_dict(with_meta=True) +print(json.dumps(validation, sort_keys=True, indent=4, separators=(',', ': '))) +``` + +## Developing rest_VariantValidator in Docker +The process for cloning the repo is the same as for VariantValidator ```bash -$ docker stop +$ cd ~/share/DevelopmentRepos +$ git clone https://github.com/openvar/rest_variantValidator.git ``` -## Delete containers +Also, branches are created in the same way ```bash -$ docker rm +$ git checkout develop +$ git pull +$ git branch name_of_branch +$ git checkout name_of_branch ``` -## Delete images +Navigating to the Repo is identical + ```bash -$ docker rmi -``` \ No newline at end of file +$ docker-compose exec restvv bash +$ cd /usr/local/share/DevelopmentRepos/rest_variantValidator +``` + +However, instead of running `pip install -e .`, we can test the install using the Python development server + +```bash +python rest_variantValidator/app.py +``` + +## Updating rest_variantValidator +To update a container, use + +```bash +$ docker-compose build --build +``` +where is a service listed in the docker-compose.yml + +Once re-built, start all containers as normal + +## Deleting rest_variantValidator + +```bash +# Remove the specific containers +$ docker-compose rm + +# OR Delete all containers on your system +$ docker-compose down +$ docker system prune -a --volumes +``` + +Run the following command to make sure all rest_validator containers are stopped +```bash +$ docker ps +``` + +Stop any individual containers by running + +```bash +$ docker stop +``` + +Then re-run the docker system prune command + +***If you choose this option, make sure you see the container restvv being re-created and all Python packages being +reinstalled in the printed logs, otherwise the container may not actually be rebuilt and the contained modules may not + update*** diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 2fee2c9e..e215a162 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -6,22 +6,20 @@ For any other systems, or if you cannot install the databases, we recommend inst ## Pre-requisites Installation requires [VariantValidator](https://github.com/openvar/variantValidator) and [VariantFormatter](https://github.com/openvar/variantFormatter) -## Download the source code +## Installing -To download the source code simply clone the master branch. - -``` +Download the git repo +```bash $ git clone https://github.com/openvar/rest_variantValidator $ cd rest_variantValidator ``` -## Python 3.6 environment - -When installing we recommend using a virtual environment, as it requires specific versions of several libraries including python and sqlite. See the [VariantValidator](https://github.com/openvar/variantValidator) installation documentation - -## Installing rest_variantValidator - -To install rest_ariantValidator within your virtual environment run: -``` -$ python setup.py install +Create a virtual environment - recommended +```bash +$ conda env create -f environment.yml +$ conda activate vvrest ``` + +See the [VariantValidator](https://github.com/openvar/variantValidator) installation documentation to install the +databases and set up configurations. You will need to run the confuguration script if you have not installed this API +previously. Contact admin if you are having any difficulties diff --git a/docs/MANUAL.md b/docs/MANUAL.md index bfd9d91e..7df9de2d 100644 --- a/docs/MANUAL.md +++ b/docs/MANUAL.md @@ -4,13 +4,13 @@ To run rest_variantValidator -### Python +### In dev mode upsing Python ```bash -$ python rest_variantValidator/wsgi.py +$ python wsgi.py ``` -You will be provided with a link which will open rest_variantValidator in your web browser. +You will be provided with a link which will open rest_variantValidator in your web browser. [http://127.0.0.1:5000/](http://127.0.0.1:5000/) ## Swagger documented functions @@ -30,37 +30,45 @@ Mounting rest_variantValidator to an Apache web server requires [mod_wsgi](https Example [Apache configuration](https://modwsgi.readthedocs.io/en/develop/user-guides/quick-configuration-guide.html) +***Note: you will need to configure the file paths in the example below*** + ```apacheconf - WSGIPythonPath /lib:/site-packages + WSGIPythonPath /local/miniconda3/envs/vvenv/lib:/local/miniconda3/envs/vvenv/lib/python3.6/site-packages WSGIDaemonProcess rest_variantValidator user=wwwrun group=www threads=5 - WSGIScriptAlias / /rest_variantValidator/wsgi.py - WSGIPythonHome + WSGIScriptAlias / /local/py3Repos/rest_variantValidator/wsgi.py + WSGIPythonHome /local/miniconda3/envs/vvenv - /rest_variantValidator/> + WSGIProcessGroup rest_variantValidator WSGIApplicationGroup %{GLOBAL} - Order deny,allow - Allow from all + Order allow,deny + Allow from all + Header set Access-Control-Allow-Origin "*" + Header set Access-Control-Allow-Methods "GET" - +LogLevel crit CustomLog /local/apache2/log/access_log for_pound -``` - -## Run in dev mode -To run rest_variantValidator on a dev server -```bash -$ python rest_variantValidator/app.py ``` -In a web-browser navkgate to `0.0.0.0:5000` +## Flask configuration + +rest_variantValidator is a Flask application. Flask can be +[configured](https://flask.palletsprojects.com/en/stable/config/) +using its +[prefixed environment variable](https://flask.palletsprojects.com/en/stable/config/#configuring-from-environment-variables) +configuration mechanism, so documented settings can be set with a prefix +of `FLASK_`. -Exit the app by holding `ctrl + c` +This mechanism is also used for Flask extensions such as the +[Flask-Limiter](https://github.com/alisaifee/flask-limiter/tree/master) +rate limiter this uses, which can be disabled by setting +`FLASK_RATELIMIT_ENABLED=`. ## Additional resources -We are compiling a number of jupyter notebook user guides for rest_variantValidator in [rest_variantValidator_manuals](https://github.com/openvar/rest_variantValidator_manuals) \ No newline at end of file +We are compiling a number of jupyter notebook user guides for rest_variantValidator in [rest_variantValidator_manuals](https://github.com/openvar/rest_variantValidator_manuals) diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md new file mode 100644 index 00000000..bbbdcb1a --- /dev/null +++ b/docs/USER_MANUAL.md @@ -0,0 +1,204 @@ +## 1. Request a User Account via Swagger API + +The VariantValidator API provides a Swagger interface to submit account requests interactively or via JSON requests. + +1. Navigate to [https://www183.lamp.le.ac.uk/](https://www183.lamp.le.ac.uk/) +2. Locate the **`POST /auth/request_account`** endpoint. +3. Click **"Try it out"**. +4. Fill in the fields in the form: + +- **Username** – your desired username (3–20 characters: letters, numbers, underscores, dots, or dashes). +- **Primary Email** – your main email address. If this is not a workplace email, you must provide additional information in the optional fields to help verify your identity. +- **Professional Email** – optional, use if you have a work email. +- **ORCID** – optional ORCID identifier. +- **Link to Company Profile** – optional link to a public company profile or other identifying information. + +**Important:** Users must provide sufficient identifying information, particularly if the primary email is not a workplace email. Insufficient information may result in the account request being rejected. + +5. Click **"Execute"** to submit the request. + +Once submitted, your account request will be reviewed by an administrator. You will receive an email notification once the account is approved or rejected. + +--- + +## 2. Manage Your Account + +Once your account has been approved, you can reset your password, generate new access tokens, and view your account details. The API supports several authentication methods for scripts and CLI: + +- **Basic Auth** – username and password in the HTTP header (standard user/password authentication). +- **Bearer Token Auth** – using the token returned from `/auth/new_token`. +- **JSON body** – for endpoints that accept credentials in the body (e.g., `/auth/reset_password`). +- **Query parameters** – for Swagger-generated curl commands. + +--- + +### 2.1 Set a New Password + +Endpoint: `/auth/reset_password` + +```bash +curl -X POST https://www183.lamp.le.ac.uk/auth/reset_password \ + -H "Content-Type: application/json" \ + -d '{ + "username": "your_username", + "current_password": "current_password_here", + "new_password": "new_secure_password_here" + }' +``` + +- You must provide your current password to set a new one. +- Passwords should be strong (letters, numbers, symbols). + +--- + +### 2.2 Generate a New Token + +Endpoint: `/auth/new_token` + +Authenticate using Basic Auth to generate a Bearer token: + +```bash +curl -X POST https://www183.lamp.le.ac.uk/auth/new_token \ + -u 'your_username:your_password' +``` + +Response: + +```json +{ + "your_token": "", + "token_type": "Bearer" +} +``` + +You can now use this token for endpoints that require authentication. + +--- + +### 2.3 View Your Account Details + +Endpoint: `/auth/myaccount` + +#### a) Standard Basic Auth (user/password) + +```bash +curl -v -X POST https://www183.lamp.le.ac.uk/auth/myaccount \ + -u 'your_username:your_password' +``` + +#### b) Bearer Token + +```bash +curl -v -X POST https://www183.lamp.le.ac.uk/auth/myaccount \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " +``` + +#### c) JSON body (for endpoints that accept it) + +```bash +curl -X POST https://www183.lamp.le.ac.uk/auth/myaccount \ + -H "Content-Type: application/json" \ + -d '{ + "username": "your_username", + "current_password": "your_password" + }' +``` + +#### d) Query parameters (for Swagger cURL) + +```bash +curl -X POST "https://www183.lamp.le.ac.uk/auth/myaccount?username=your_username¤t_password=your_password" +``` + +Response: + +```json +{ + "access_token": "", + "token_type": "Bearer", + "expires_in_days": 365 +} +``` + +--- + +**Notes:** + +- The API checks authentication methods in the following order: Bearer token → Basic Auth → JSON body → Query params. +- If you lose your token, reset it via `/auth/reset_password`. +- Keep your tokens and passwords secure. Tokens are valid until they expire, as indicated in `expires_in_days`. + +--- + +## 3 Workflow Endpoint Examples + +### 3.1 Curl Examples + +- Using Bearer token for a workflow endpoint: + +```bash +curl -X POST https://www183.lamp.le.ac.uk/workflow/run_analysis \ + -H "Authorization: Bearer " \ + -d '{"sample_id": "12345"}' +``` + +- Using standard username/password Basic Auth for a workflow endpoint: + +```bash +curl -X POST https://www183.lamp.le.ac.uk/workflow/run_analysis \ + -u 'your_username:your_password' \ + -d '{"sample_id": "12345"}' +``` + +### 3.2 Python `requests` Examples + +You can also call the workflow endpoints from Python using the `requests` library. + +- **Using Bearer Token** + +```python +import requests + +url = "https://www183.lamp.le.ac.uk/workflow/run_analysis" +headers = { + "Authorization": "Bearer ", + "Content-Type": "application/json" +} +data = { + "sample_id": "12345" +} + +response = requests.post(url, headers=headers, json=data) +print(response.status_code) +print(response.json()) +``` + +- **Using Basic Auth (username/password)** + +```python +import requests +from requests.auth import HTTPBasicAuth + +url = "https://www183.lamp.le.ac.uk/workflow/run_analysis" +data = { + "sample_id": "12345" +} + +response = requests.post(url, auth=HTTPBasicAuth('your_username', 'your_password'), json=data) +print(response.status_code) +print(response.json()) +``` + +**Notes:** + +- Make sure to replace `` or `'your_username'` / `'your_password'` with your actual credentials. +- `requests` automatically handles HTTPS connections. +- For large workflows or multiple requests, consider using a session to reuse connections: + +```python +with requests.Session() as session: + session.auth = HTTPBasicAuth('your_username', 'your_password') + response = session.post(url, json=data) + print(response.json()) +``` diff --git a/docs/images/curl.png b/docs/images/curl.png new file mode 100644 index 00000000..427ab6db Binary files /dev/null and b/docs/images/curl.png differ diff --git a/docs/images/ns_and_ep.png b/docs/images/ns_and_ep.png new file mode 100644 index 00000000..8a0d011b Binary files /dev/null and b/docs/images/ns_and_ep.png differ diff --git a/docs/images/query_builder.png b/docs/images/query_builder.png new file mode 100644 index 00000000..30378811 Binary files /dev/null and b/docs/images/query_builder.png differ diff --git a/docs/images/response.png b/docs/images/response.png new file mode 100644 index 00000000..f2f576e5 Binary files /dev/null and b/docs/images/response.png differ diff --git a/docs/images/urls.png b/docs/images/urls.png new file mode 100644 index 00000000..283faf2a Binary files /dev/null and b/docs/images/urls.png differ diff --git a/docs/images/web.png b/docs/images/web.png new file mode 100644 index 00000000..9273ac50 Binary files /dev/null and b/docs/images/web.png differ diff --git a/docs/rest_variantValidator_tutorial.ipynb b/docs/rest_variantValidator_tutorial.ipynb new file mode 100644 index 00000000..f3972ee1 --- /dev/null +++ b/docs/rest_variantValidator_tutorial.ipynb @@ -0,0 +1,586 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + "
Lecturer - Healthcare Sciences
\n", + "
(Clinical Bioinformatics)
\n", + "
The University of Manchester
\n", + " 		\n", + " \n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to SPRINT 1 notebook B\n", + "****\n", + "\n", + "## Overview\n", + "This introductory notebook, and subsequent SPRINT_introduction notebooks will continue to introduce the concept of retrieving data from Application Programming Interfaces [API](https://en.wikipedia.org/wiki/Application_programming_interface) which are web-hosted ([web API](https://en.wikipedia.org/wiki/Web_API))\n", + "\n", + "Many bioinformatics tools and data repositories can be accessed using web APIs including NCBI and Ensembl. \n", + "\n", + "Although we cannot hope to demonstrate how each an every useful bioinformatics web API works during this 10 week course, we will give you a broad overview of the tools we use to request data from these resources and the tools we use to make-sense of the data that are returned " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Table of Contents

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Notebook B, section 1: [Introduction to JSON](#json)\n", + "- [What is JSON](#watsit)\n", + "- [The JSON format](#jform)\n", + "- [Reading and writing JSON using Python](#jpy)\n", + "- [Section 1 Summary](#s1s)\n", + "\n", + "#### Notebook B, section 2: [Introduction to REST API](#restapi)\n", + "- [The REST framework](#rest)\n", + "- [Building a simple API: Part A - Build a simple REST API](#builder_a)\n", + "- [Building a simple API: Part B - Request data using Python](#builder_b)\n", + "- [Building a simple API: Part C - Create a new VariantValidator API](#builder_c)\n", + "- [Section 2 Summary and Assignment](#s2s)\n", + "- [Marked assessment](#practical)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
Learning Objective: Create functioning, standards compliant and well documented Python code
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "

Introduction to the VariantValidator REST API

\n", + " 		\n", + " \n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "Image by Peter Causey-Freeman" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

The REST framework

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [An Introduction to APIs](https://restful.io/an-introduction-to-api-s-cee90581ca1b) \n", + "- [Gonzalo Vázquez](https://restful.io/@gonzalovazquez)\n", + "- [Restful Web](https://restful.io/) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Using the VariantValidator REST API

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### API structure\n", + "\n", + "#### Framework\n", + "The VariantValidator REST API is built on the [Flask](https://en.wikipedia.org/wiki/Flask_\\(web_framework) web framework. \n", + "\n", + "The REST components are built using [Flask-RESTPlus](https://flask-restplus.readthedocs.io/en/stable/)\n", + "\n", + "> Flask-RESTPlus is an extension for Flask that adds support for quickly building REST APIs. Flask-RESTPlus encourages best practices with minimal setup. \n", + "\n", + ">It provides a coherent collection of tools to describe your API and expose its documentation properly (using Swagger).\n", + "\n", + "#### Namespaces and Endpoints\n", + "\n", + "The VariantValidator REST API has several tool-sets. Each set is divided into separate namespaces. \n", + "\n", + "For example, the namespace \"hello\" is used to test whether our services are up-and running. The namespaces and endpoints are most easily demonstrated by looking at the Swagger documented API on [https://rest.variantvalidator.org/](https://rest.variantvalidator.org/).\n", + "\n", + "The namespaces are\n", + "- VariantValidator; Core [VariantValidator](https://github.com/openvar/variantValidator) Python library\n", + "- VariantFormatter; [VariantFormatter](https://github.com/openvar/variantFormatter/tree/develop) extension library\n", + "- LOVD; Adapted endpoint for LOVD specific access to our resourced\n", + "- hello; Simple handshake allowing external users to test whether services are alive before submission\n", + "\n", + "Swagger documentation displays the namespaces as follows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![title](images/ns_and_ep.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each namespace contains endpoints which access specific functions of the VariantValidator libraries. For example the VariantValidator namespace has 3 endpoints\n", + "- gene2transcripts\n", + "- hgvs2reference\n", + "- variantvalidator\n", + "\n", + "### Building Queries\n", + "\n", + "In this interactive mode, the endpoint can be clicked allowing us to access a human-friendly query builder\n", + "\n", + "![title](images/query_builder.png)\n", + "\n", + "Currently the data can be returned in 2 different formats, JSON and XML. These are selected using the `Select the response format` drop-down menu. \n", + "\n", + "For this example I have selected the simple `gene2transcripts` endpoint which searches for all transcripts associated with a particular gene. The documentation tells us that me must input either a HGNC compliant gene symbol or a RefSeq transcript ID. However, this documentation will be improved because the tool also accepts RefSeq transcript IDs without version numbers, LRG IDs (*e.g.* LRG_1) and LRG transcript IDs (*e.g.* LRG_1t1). \n", + "\n", + "Once all the required fields are populated we can execute the query" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The API response\n", + "\n", + "Let's take a look at the response which Swagger has parsed into a user-friendly web page.\n", + "\n", + "![title](images/response.png)\n", + "\n", + "#### Server responses\n", + "1. [Response code](https://developer.amazon.com/docs/amazon-drive/ad-restful-api-response-codes.html) 200\n", + "2. The Response headers provide additional response metadata, *e.g.* the content-type and the time of the response\n", + "3. Response body, *i.e.* the JSON or XML the endpoint returns\n", + "\n", + "### The API query URLS\n", + "\n", + "Swagger also displays queries that can be used to trigger the response in a standard format, *i.e.* a non-interactive mode.\n", + "\n", + "![title](images/urls.png)\n", + "\n", + "#### curl\n", + "\n", + "curl is generally used in terminals and programming\n", + "\n", + "In this screen shot I have used a terminal to request data directly from the VariantValidator API using the provided curl. I have piped this into `python -m json.tool` to provide a pretty JSON display.\n", + "\n", + "The full request is `curl -X GET \"https://rest.variantvalidator.org/VariantValidator/tools/gene2transcripts/COL1A1?content-type=application%2Fjson\" -H \"accept: application/json\" | python -m json.tool`\n", + "\n", + "![title](images/curl.png)\n", + "\n", + "#### web URL\n", + "The web URL can simply be pasted into a browser and in the next section we will use the web URL to recover data from the VariantValidator API using Python\n", + "\n", + "![title](images/web.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Request data using Python

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The Requests module\n", + "[Requests](https://2.python-requests.org/en/master/)\n", + "> Requests: HTTP for Humans™\n", + "\n", + "> Requests is the only HTTP library for Python safe for human consumption\n", + "\n", + "***\n", + "Courtesy of the \"requests\" © 2019 Kenneth Reitz [Apache 2 License](https://www.apache.org/licenses/LICENSE-2.0)
\n", + "\n", + "OK, we have to take their word for it, but we are going to use requests because is's simple, easy to understand and is well maintained\n", + "\n", + "#### Method\n", + "\n", + "1. Install requests into your environment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install requests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "2. Import modules we will use" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "3. Create a simple function that calls the API using responses\n", + "\n", + " - *Note: This function is in a format that can be expanded*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "base_url = 'http://127.0.0.1:5000/'\n", + "def make_request(base_url, api_function):\n", + " # Tell the User the full URL of their call to the rest API\n", + " url = '%s%s' % (base_url, api_function)\n", + " print(\"Querying rest API with URL: \" + url)\n", + " \n", + " # Make the request and pass to a response object that the function returns\n", + " response = requests.get(url)\n", + " return response" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "4. Make a request to our API using the function. We need to specify the base_url and the api_function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = make_request(base_url, 'hello')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "5. Look at the response content\n", + " - response status code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response.status_code" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " - response headers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response.headers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "5. Finally, extract the body which the requests.json() method formats into Python dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "body = response.json()\n", + "body" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Building a simple API: Part C - Create a new VariantValidator API

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A bit basic isn't it?\n", + "\n", + "The simple `hello` API is a bit basic, but it does show you how an API works and we have also made requests to our API using Python.\n", + "\n", + "So what if we want to pass some data to the API?\n", + "\n", + "To `application/app_v2` I have added an additional **namespace** called name_space.\n", + "\n", + "I have also added a new API to our REST interface called name\n", + "\n", + "```python\n", + "name_space = application.namespace('name', description='Return a name provided by the user')\n", + "@name_space.route(\"\")\n", + "class NameClass(Resource):\n", + " def get(self, name):\n", + " return {\n", + " \"My name is\" : name\n", + " }\n", + "```\n", + "\n", + "To capture data submitted to the API, we tell the name_space.route to expect a **string** object\n", + "```python\n", + "@name_space.route(\"/\")\n", + "```\n", + "\n", + "and the NameClass Resource to expect the string object\n", + "```python\n", + "def get(self, name):\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "

Have a go

\n", + "\n", + "Activate `app_v2`\n", + "\n", + "```bash\n", + "$ python SPRINT/application/app_v2.py\n", + "```\n", + "\n", + "\n", + "### Exercise\n", + "\n", + "In a browser navigate to [http://127.0.0.1:5000/](http://127.0.0.1:5000/) and see whether you can figure out how to return your name using the API\n", + "\n", + "*Swagger is your friend here. It makes it very simple for a lay user to use an API*\n", + "\n", + "
\n", + "\n", + "### Exercise 2\n", + "\n", + "Now write a script that can make a call to the API and return the JSON that displays your name\n", + "\n", + "*Use the script above as a template. Remember, you may want to make a call to the hello API again, so keep the function flexible*\n", + "\n", + "Once the script is working, print out the response status, headings and JSON\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "

Section 2 Summary and Assignment

 \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Summary\n", + "In section 2 of this notebook we have learned about the REST API framework. We have learned how to build a simple REST API of our own. We have briefly touched upon the concept of how Swagger documentation makes APIs accessible to mere humans\\*. We have also learned how to request and make sense of data returned by REST PAIs using the Python requests module\n", + "\n", + "\\**We will look at of Swagger in more detail in week 8 of this unit* " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_style": "split" + }, + "source": [ + "\n", + "\n", + "### Over to you\n", + "\n", + "#### Aim of this exercise\n", + "The aim of this exercise is to keep you into the mindset of working together as a team. We will concentrate on aspects of working in an Agile fashion.\n", + "\n", + "#### Structure your team\n", + "Assign your team roles:\n", + "\n", + "1. **Project lead**\n", + " - Initiate the project on Git Issues (Note, there are two separate short projects here)\n", + " - Lead the group discussion in Git Issues and Slack\n", + " - Provide final feedback on the group's activities and close the issue\n", + " \n", + "\n", + "2. **Team members**\n", + " - Coders who will be responsible for writing the Python functions\n", + " - Testers who will be responsible for testing the code and providing feedback to the coders\n", + "\n", + "***We recommend ensuring that you most experienced coders work with your least experienced coders. Don't forget, this is a team assignment, if you can't figure out how to do something, ask your team on Slack!***" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_style": "split" + }, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_style": "split" + }, + "source": [ + "### Work-flow\n", + "\n", + "1. Group leader creates an issue on Git Issues\n", + "2. The coders will work together to write the module\n", + "3. The testers will review the final code and test the code. Feedback will be given to the coders within the Git issue\n", + "4. Once the coding is completed and tested, the project lead will summarise the key work-flow points and close the issue\n", + "\n", + "**Details about how the assignment will be marked can be found [here](LINK)**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "### Team Assignment\n", + "\n", + "***Remember, you are working as a team. Make sure you assign tasks in an agile way***\n", + "\n", + "#### Coding Workflow\n", + "1. In `applications/app_v3.py` I have created a `vv_space` namespace and `VariantValidatorClass` resource (Endpoint). Your task is to replace all the sections of the module marked \\_\\_\\_\\_\\_ (5 underscores) with actual code. The namespace requires 3 variables.\n", + "\n", + "When you have finished filling in the blanks, the answers can be found in `app_v4.py`\n", + "\n", + "
\n", + "\n", + "***Refer to the existing [VariantValidator REST API](https://rest.variantvalidator.org/webservices/variantvalidator.html#!/variantvalidator/VariantValidator)***\n", + "\n", + "2. In `applications/app_v3.py` create a new namespace and Endpoint that incorporates and returns the data from the function you created in SPRINT_1_introduction_a. When you are creating the namespace route, add a field that allows the user to select whether or not the sequences your function returns are displayed.\n", + " \n", + "*Note: for non-coding transcripts some of these fields will need to return None*\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Concluding remarks\n", + "We will cover methods for reading and writing JSON data to-and-from files in week_6, but a key aspect of learning to program is learning to use the internet to find out how to do things. Google and stack overflow are your friends!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..3a4438ab --- /dev/null +++ b/environment.yml @@ -0,0 +1,8 @@ +name: vvrest +channels: + - conda-forge + - bioconda +dependencies: + - python==3.12.11 + - pip + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fb3666b4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,96 @@ +# Project metadata +[project] +name = "rest_VariantValidator" +dynamic = ["version"] # Use dynamic version from setuptools_scm +description = "REST API interface for VariantValidator" +license = "AGPL-3.0-only" +license-files = ["LICENSE.txt"] +authors = [{name = "VariantValidator Contributors", email = "admin@variantvalidator.org"}] +readme = "README.md" +keywords = [ + "bioinformatics", + "computational biology", + "genome variants", + "genome variation", + "genomic variants", + "genomic variation", + "genomics", + "hgvs", + "HGVS", + "sequencevariants" +] +requires-python = ">=3.6" + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] + +# List of project dependencies +dependencies = [ + # Dependencies that will be installed via PyPi + "httplib2", + "configparser", + "dicttoxml", + "gunicorn", + "flask-restx", + "Flask", + "Jinja2", + "Werkzeug", + "MarkupSafe", + "flask-cors", + "flask_httpauth", + "flask_limiter", + + # Dependencies from other repositories, specified with their repository URLs and package names + "vvhgvs@git+https://github.com/openvar/vv_hgvs@master", + "VariantFormatter@git+https://github.com/openvar/variantFormatter@master", + "VariantValidator@git+https://github.com/openvar/variantValidator@master" +] + +# URLs related to the project +[project.urls] +Homepage = "https://variantvalidator.org/" +Source = "https://github.com/openvar/rest_variantValidator" +"Bug Reports" = "https://github.com/openvar/variantValidator/issues" +"Say Thanks!" = "https://www.buymeacoffee.com/VariantValidatr" + +# Console scripts exposed by the package +[scripts] +update_vdb = "bin/update_vdb:main" +variant_validator = "bin/variant_validator:main" +vv_configure = "bin/vv_configure:main" + +# Additional data files to include in the package +data = [ + { include = "configuration", glob = "configuration/empty_vv_db.sql" } +] + +# setuptools SCM for version management +[tool.setuptools_scm] + +# Package discovery configuration +[tool.setuptools.packages.find] +where = ["."] +include = ["rest_VariantValidator*"] +exclude = ["batch", "locust"] + +# Build system configuration +[build-system] +requires = [ + "setuptools>=45", + "setuptools_scm[toml]>=6.2", + "wheel" +] +build-backend = "setuptools.build_meta" + +# Pytest configuration +[tool.pytest.ini_options] +testpaths = ["tests"] \ No newline at end of file diff --git a/rest_VariantValidator/__init__.py b/rest_VariantValidator/__init__.py new file mode 100644 index 00000000..09841b4d --- /dev/null +++ b/rest_VariantValidator/__init__.py @@ -0,0 +1,31 @@ +import importlib.metadata +import re +import warnings + +# Pull in use_scm_version=True enabled version number +_is_released_version = False +try: + __version__ = importlib.metadata.version("rest_VariantValidator") + if re.match(r"^\d+\.\d+\.\d+$", __version__) is not None: + _is_released_version = True +except importlib.metadata.PackageNotFoundError: + warnings.warn("can't get __version__ because VariantValidator package isn't installed", Warning) + __version__ = None + + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/app.py b/rest_VariantValidator/app.py new file mode 100644 index 00000000..b95466c0 --- /dev/null +++ b/rest_VariantValidator/app.py @@ -0,0 +1,216 @@ +# Import modules +from flask import Flask, request +from rest_VariantValidator.endpoints import api +from flask_cors import CORS +from rest_VariantValidator.utils import exceptions, request_parser, representations +from logging import handlers +import time +import os +from pathlib import Path +import logging.config +from configparser import ConfigParser +from VariantValidator import settings as vv_settings +from rest_VariantValidator.utils.limiter import limiter +from werkzeug.exceptions import InternalServerError, Forbidden, TooManyRequests + +# Set document root +ROOT = os.path.dirname(os.path.abspath(__file__)) +path = Path(ROOT) +parent = path.parent.absolute() + +# Change settings based on config +config = ConfigParser() +config.read(vv_settings.CONFIG_DIR) + +""" +Logging +""" +if config['logging'].getboolean('log') is True: + logger = logging.getLogger('rest_VariantValidator') + console_level = config['logging']['console'].upper() + log_console_level = logging.getLevelName(console_level) + logger.setLevel(log_console_level) + + try: + parent = config['logging']['file_path'] + except KeyError: + pass + logHandler = handlers.RotatingFileHandler(str(parent) + '/rest_VariantValidator.log', + maxBytes=500000, + backupCount=2) + + file_level = config['logging']['file'].upper() + log_file_level = logging.getLevelName(file_level) + logHandler.setLevel(log_file_level) + logger.addHandler(logHandler) + +""" +Create a parser object locally +""" +parser = request_parser.parser + +# Define the application as a Flask app with the name defined by __name__ +application = Flask(__name__) +application.config.from_prefixed_env() + +# Create a limiter instance and attach it to your Flask application +limiter.init_app(application) +api.init_app(application) + +# By default, show all endpoints (collapsed) +application.config.SWAGGER_UI_DOC_EXPANSION = 'list' + +# enable CORS +CORS(application, resources={r'/*': {'origins': '*'}}) + +""" +Representations +""" +@api.representation('text/xml') +def application_xml(data, code, headers): + resp = representations.xml(data, code, headers) + resp.headers['Content-Type'] = 'text/xml' + return resp + + +@api.representation('application/json') +def application_json(data, code, headers): + resp = representations.application_json(data, code, headers) + resp.headers['Content-Type'] = 'application/json' + return resp + +""" +Error handlers +""" + + +def log_exception(exception_type): + params = dict(request.args) + params['path'] = request.path + message = '%s occurred at %s with params=%s\n' % (exception_type, time.ctime(), params) + logger.exception(message, exc_info=True) + + +@application.errorhandler(exceptions.RemoteConnectionError) +def remote_connection_error_handler(e): + log_exception('ApplicationRemoteConnectionError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return application_json({'message': str(e)}, + 504, + None) + else: + return application_xml({'message': str(e)}, + 504, + None) + + +@application.errorhandler(404) +def not_found_error_handler(e): + log_exception('ApplicationNotFoundError') + logger.warning(str(e)) + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return application_json({'message': 'Requested Endpoint not found: See the documentation at https://rest.variantvalidator.org'}, + 404, + None) + else: + return application_xml({'message': 'Requested Endpoint not found: See the documentation at https://rest.variantvalidator.org'}, + 404, + None) + + +@application.errorhandler(500) +def internal_server_error_handler(e): + log_exception('ApplicationInternalServerError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return application_json({'message': 'Unhandled error: contact https://variantvalidator.org/contact_admin/'}, + 500, + None) + else: + return application_xml({'message': 'Unhandled error: contact https://variantvalidator.org/contact_admin/'}, + 500, + None) + + +@application.errorhandler(429) +def too_many_requests_error_handler(e): + log_exception('ApplicationTooManyRequestsError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return application_json({'message': 'Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org'}, + 429, + None) + else: + return application_xml({'message': 'Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org'}, + 429, + None) + + +@application.errorhandler(403) +def forbidden_error_handler(e): + log_exception('ApplicationForbiddenError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return application_json({'message': 'Forbidden: You do not have the necessary permissions.'}, + 403, + None) + else: + return application_xml({'message': 'Forbidden: You do not have the necessary permissions.'}, + 403, + None) + + +@api.errorhandler(InternalServerError) +def api_internal_server_error_handler(e): + log_exception('APIInternalServerError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return {'message': 'Unhandled error: contact https://variantvalidator.org/contact_admin/'}, 500, {'Content-Type': 'application/json'} + else: + return {'message': 'Unhandled error: contact https://variantvalidator.org/contact_admin/'}, 500, {'Content-Type': 'text/xml'} + + +@api.errorhandler(TooManyRequests) +def api_too_many_requests_error_handler(e): + log_exception('APITooManyRequestsError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return {'message': 'Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org'}, 429, {'Content-Type': 'application/json'} + else: + return {'message': 'Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org'}, 429, {'Content-Type': 'text/xml'} + + +@api.errorhandler(Forbidden) +def api_forbidden_error_handler(e): + log_exception('APIForbiddenError') + args = parser.parse_args() + if args['content-type'] != 'text/xml': + return {'message': 'Forbidden: You do not have the necessary permissions.'}, 403, {'Content-Type': 'application/json'} + else: + return {'message': 'Forbidden: You do not have the necessary permissions.'}, 403, {'Content-Type': 'text/xml'} + + +# Allows app to be run in debug mode +if __name__ == '__main__': + port = int(os.environ.get('PORT', 5000)) + application.debug = True + application.run(host="127.0.0.1", port=port) + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/endpoints/__init__.py b/rest_VariantValidator/endpoints/__init__.py new file mode 100644 index 00000000..49c3025e --- /dev/null +++ b/rest_VariantValidator/endpoints/__init__.py @@ -0,0 +1,107 @@ +import rest_VariantValidator +import VariantValidator +import VariantFormatter +from flask_restx import Api +from flask import url_for +from .variantvalidator_endpoints import api as ns_vv +from .variantformatter_endpoints import api as ns_vf +from .lovd_endpoints import api as ns_lovd +from .hello import api as ns_hello +#attempt to pull in and use/document auth api if it is available +try: + + from VariantValidator_APIs.db_auth.auth_endpoints import auth_api as ns_auth + # Set auth for API + authorizations = { + 'apikey': { + 'type': 'apiKey', + 'in': 'header', + 'name': 'Authorization' + }, + 'basic_pwd': { + 'type': 'basic', + 'name': 'VV_API_PWD' + } + } + security = ['apikey', 'basic_pwd'] + sec_descripton = ''' +## Security +For now the Swagger documented endpoints retain the last entered login even on +page refresh, at least on some browsers, to "log out" please enter a trivial +invalid login e.g. username:none password: none to overwrite this. + +For logging in via a token please prefix your token with "Bearer " (including +the space). +''' +except ModuleNotFoundError: + ns_auth = None + authorizations = None + security = None + sec_descripton ='' + +# Obtain VariantValidator related metadata +vval = VariantValidator.Validator() +config_dict = vval.my_config() + + +# Override standard specs_url to allow reverse-proxy access through mod_wsgi +class CustomAPI(Api): + @property + def specs_url(self): + """ + The Swagger specifications absolute url (ie. `swagger.json`) + + This method returns the path relative to the APP required for reverse proxy access + + :rtype: str + """ + return url_for(self.endpoint('specs'), _external=False) + + +# Define the API as api +api = CustomAPI(version=rest_VariantValidator.__version__, + title="rest_VariantValidator", + description="## By continuing to use this service you agree to our terms and conditions of Use\n" + "- [Terms and Conditions](https://github.com/openvar/variantValidator/blob" + "/master/README.md)\n\n" + "## Powered by\n" + "- [VariantValidator](https://github.com/openvar/rest_variantValidator) version " + + VariantValidator.__version__ + "\n" + "- [VariantFormatter](https://github.com/openvar/variantFormatter) version " + + VariantFormatter.__version__ + "\n" + " - [vv_hgvs](https://github.com/openvar/vv_hgvs) version " + + config_dict['variantvalidator_hgvs_version'] + "\n" + " - [VVTA](https://www528.lamp.le.ac.uk/) release " + + config_dict['vvta_version'] + "\n" + " - [vvSeqRepo](https://www528.lamp.le.ac.uk/) release " + + config_dict['vvseqrepo_db'].split('/')[-2] + sec_descripton, + authorizations=authorizations, + security=security + ) + +# Add the namespaces to the API +api.add_namespace(ns_vv) +api.add_namespace(ns_vf) +api.add_namespace(ns_lovd) +api.add_namespace(ns_hello) +if ns_auth is not None: + api.add_namespace(ns_auth, path='/auth') # Mount auth endpoints under /auth + + + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/endpoints/hello.py b/rest_VariantValidator/endpoints/hello.py new file mode 100644 index 00000000..ea81e7b7 --- /dev/null +++ b/rest_VariantValidator/endpoints/hello.py @@ -0,0 +1,107 @@ +from flask_restx import Namespace, Resource +from rest_VariantValidator.utils import request_parser, representations, exceptions +from rest_VariantValidator.utils.limiter import limiter +from flask import abort +from rest_VariantValidator.utils.object_pool import vval_object_pool + +""" +Create a parser object locally +""" +parser = request_parser.parser + + +""" +The assignment of api changes +""" + +api = Namespace('hello', description='Endpoint to check services are "alive" and display the current software and ' + 'database versions') + +""" +We also need to re-assign the route ans other decorated functions to api +""" + + +@api.route("/", strict_slashes=False) +class HelloClass(Resource): + + # Add documentation about the parser + @api.expect(parser, validate=True) + def get(self): + + # Import object from vval pool + vval = vval_object_pool.get_object() + + # Collect Arguments + args = parser.parse_args() + config_dict = vval.my_config() + config_dict['vvseqrepo_db'] = config_dict['vvseqrepo_db'].split('/')[-2] + + # Return object to vval pool + vval_object_pool.return_object(vval) + + # Overrides the default response route so that the standard HTML URL can return any specified format + if args['content-type'] == 'application/json': + # example: http://127.0.0.1:5000/name/name/bob?content-type=application/json + return representations.application_json({ + "status": "hello_world", + "metadata": config_dict + }, + 200, None) + # example: http://127.0.0.1:5000/name/name/bob?content-type=text/xml + elif args['content-type'] == 'text/xml': + return representations.xml({ + "status": "hello_world", + "metadata": config_dict + }, + 200, None) + else: + # Return the api default output + return { + "status": "hello_world", + "metadata": config_dict + } + +@api.route('/limit') +class LimitedRateHelllo(Resource): + @limiter.limit("1/second") + @api.expect(parser, validate=True) + def get(self): + return { "status": "not yet hitting the rate limit" } + + +@api.route('/trigger_error/') +class ExceptionClass(Resource): + @api.expect(parser, validate=True) + def get(self, error_code): + if error_code == 400: + abort(400) + elif error_code == 403: + abort(403) + elif error_code == 404: + abort(404) + elif error_code == 500: + abort(500) + elif error_code == 429: + abort(429) + elif error_code == 999: + raise exceptions.RemoteConnectionError('https://rest.variantvalidator.org/variantvalidator currently ' + 'unavailable') + + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/endpoints/lovd_endpoints.py b/rest_VariantValidator/endpoints/lovd_endpoints.py new file mode 100644 index 00000000..58f2bc18 --- /dev/null +++ b/rest_VariantValidator/endpoints/lovd_endpoints.py @@ -0,0 +1,140 @@ +# Import modules +import ast +from flask_restx import Namespace, Resource +from rest_VariantValidator.utils import request_parser, representations, input_formatting +from rest_VariantValidator.utils.object_pool import simple_variant_formatter_pool +from rest_VariantValidator.utils.limiter import limiter, formatter_pool_limit +# get login authentication, if needed, or dummy auth if not present +try: + from VariantValidator_APIs.db_auth.verify_password import auth +except ModuleNotFoundError: + from rest_VariantValidator.utils.verify_password import auth + + +def ordereddict_to_dict(value): + for k, v in value.items(): + if isinstance(v, dict): + value[k] = ordereddict_to_dict(v) + return dict(value) + + +""" +Create a parser object locally +""" +parser = request_parser.parser + +api = Namespace('LOVD', description='LOVD API Endpoints') + + +@api.route("/lovd////" + "//", strict_slashes=False) +@api.doc(description="This endpoint uses a dynamic rate limit based on pool availability.") +@api.param("variant_description", "***Genomic HGVS***\n" + "> - NC_000017.10:g.48275363C>A\n" + "\n***Pseudo-VCF***\n" + "> - 17-50198002-C-A\n" + "> - 17:50198002:C:A\n" + "\n> *Notes*\n" + "> - *pVCF, multiple comma separated ALTs are supported*\n " + "> - *Multiple variants can be submitted, separated by the pipe '|' character*\n" + "> - *Recommended maximum is 10 variants per submission*") +@api.param("transcript_model", "***Accepted:***\n" + "> - refseq (return data for RefSeq transcript models)\n" + "> - ensembl (return data for ensembl transcript models)\n" + "> - all") +@api.param("select_transcripts", "***Return all possible transcripts***\n" + "> None or all (all transcripts at the latest versions)\n" + "> raw (all transcripts all version)\n" + "> select (select transcripts)\n" + "> mane (MANE Select and MANE Plus Clinical transcripts)\n" + "> mane_select (MANE Select transcripts)\n" + "\n***Single***\n" + "> NM_000093.4\n" + "\n***Multiple***\n" + "> NM_000093.4|NM_001278074.1|NM_000093.3") +@api.param("genome_build", "***Accepted:***\n" + "> - GRCh37\n" + "> - GRCh38\n" + "> - hg19\n" + "> - hg38\n") +@api.param("checkonly", "***Accepted:***\n" + "> - True (return ONLY the genomic variant descriptions and not transcript and protein" + " descriptions)\n" + "> - False\n" + "> - tx (Stop at transcript level, exclude protein)") +@api.param("liftover", "***Accepted***\n" + "> - True - (liftover to all genomic loci)\n" + "> - primary - (lift to primary assembly only)\n" + "> - False") +class LOVDClass(Resource): + @api.expect(parser, validate=True) + @auth.login_required() + @limiter.limit(formatter_pool_limit) # <-- dynamic limiter + def get(self, genome_build, variant_description, transcript_model, select_transcripts, checkonly, liftover, user_id=None): + # Normalize input values + if transcript_model in ('None', 'none'): + transcript_model = None + if select_transcripts in ('None', 'none'): + select_transcripts = None + if checkonly in ('False', 'false'): + checkonly = False + if checkonly in ('True', 'true'): + checkonly = True + if liftover in ('True', 'true'): + liftover = True + if liftover in ('False', 'false'): + liftover = False + + # Get a formatter from the pool + simple_formatter = simple_variant_formatter_pool.get() + + # Convert inputs to JSON arrays + variant_description = input_formatting.format_input(variant_description) + select_transcripts = input_formatting.format_input(select_transcripts) + if select_transcripts == '["all"]': + select_transcripts = "all" + if select_transcripts == '["raw"]': + select_transcripts = "raw" + + try: + content = simple_formatter.format( + variant_description, genome_build, transcript_model, + select_transcripts, checkonly, liftover + ) + except Exception as e: + return {"error": str(e)}, 500 + finally: + simple_variant_formatter_pool.return_object(simple_formatter) + + # Convert OrderedDict to normal dict + to_dict = ordereddict_to_dict(content) + content = ast.literal_eval(str(to_dict).replace("'", '"')) + + # Parse query arguments + args = parser.parse_args() + + # Return content in requested format + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(str(content), 200, None) + else: + return content + + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/endpoints/variantformatter_endpoints.py b/rest_VariantValidator/endpoints/variantformatter_endpoints.py new file mode 100644 index 00000000..94b2383d --- /dev/null +++ b/rest_VariantValidator/endpoints/variantformatter_endpoints.py @@ -0,0 +1,118 @@ +# Import modules +from flask_restx import Namespace, Resource +from rest_VariantValidator.utils import request_parser, representations, input_formatting +from rest_VariantValidator.utils.object_pool import simple_variant_formatter_pool +from rest_VariantValidator.utils.limiter import limiter, formatter_pool_limit +# get login authentication, if needed, or dummy auth if not present +try: + from VariantValidator_APIs.db_auth.verify_password import auth +except ModuleNotFoundError: + from rest_VariantValidator.utils.verify_password import auth + +""" +Create a parser object locally +""" +parser = request_parser.parser + +api = Namespace('VariantFormatter', description='Variantformatter API Endpoints') + + +@api.route("/variantformatter////" + "/", strict_slashes=False) +@api.doc(description="This endpoint uses a dynamic rate limit based on pool availability.") +@api.param("variant_description", "***Genomic HGVS***\n" + "> - NC_000017.10:g.48275363C>A\n" + "\n***Pseudo-VCF***\n" + "> - 17-50198002-C-A\n" + "> - 17:50198002:C:A\n" + "\n> *Notes*\n" + "> - *pVCF, multiple comma separated ALTs are supported*\n " + "> - *Multiple variants can be submitted, separated by the pipe '|' character*\n" + "> - *Recommended maximum is 10 variants per submission*") +@api.param("transcript_model", "***Accepted:***\n" + "> - refseq (return data for RefSeq transcript models)\n" + "> - ensembl (return data for ensembl transcript models)\n" + "> - all") +@api.param("select_transcripts", "***Return all possible transcripts***\n" + "> None or all (all transcripts at the latest versions)\n" + "> raw (all transcripts all version)\n" + "> select (select transcripts)\n" + "> mane (MANE select transcripts)\n" + "> mane_select (MANE select and MANE Plus Clinical transcripts)\n" + "\n***Single***\n" + "> NM_000093.4\n" + "\n***Multiple***\n" + "> NM_000093.4|NM_001278074.1|NM_000093.3") +@api.param("genome_build", "***Accepted:***\n" + "> - GRCh37\n" + "> - GRCh38") +@api.param("checkonly", "***Accepted:***\n" + "> - True (return ONLY the genomic variant descriptions and not transcript and protein" + " descriptions)\n" + "> - False") +class VariantFormatterClass(Resource): + # Add documentation about the parser + @api.expect(parser, validate=True) + @auth.login_required() + @limiter.limit(formatter_pool_limit) # <-- dynamic limiter + def get(self, genome_build, variant_description, transcript_model, select_transcripts, checkonly, user_id=None): + # Normalize input values + if transcript_model in ('None', 'none'): + transcript_model = None + if select_transcripts in ('None', 'none'): + select_transcripts = None + if checkonly in ('False', 'false'): + checkonly = False + if checkonly in ('True', 'true'): + checkonly = True + + # Get a formatter from the pool + simple_formatter = simple_variant_formatter_pool.get() + + # Convert inputs to JSON arrays + variant_description = input_formatting.format_input(variant_description) + select_transcripts = input_formatting.format_input(select_transcripts) + if select_transcripts == '["all"]': + select_transcripts = "all" + if select_transcripts == '["raw"]': + select_transcripts = "raw" + if select_transcripts == '["mane"]': + select_transcripts = "mane" + if select_transcripts == '["mane_select"]': + select_transcripts = "mane_select" + + try: + content = simple_formatter.format(variant_description, genome_build, transcript_model, + select_transcripts, checkonly) + except Exception as e: + return {"error": str(e)}, 500 + finally: + simple_variant_formatter_pool.return_object(simple_formatter) + + # Parse query arguments + args = parser.parse_args() + + # Return content in requested format + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(content, 200, None) + else: + return content + +# +# Copyright (C) 2016-2021 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/endpoints/variantvalidator_endpoints.py b/rest_VariantValidator/endpoints/variantvalidator_endpoints.py new file mode 100644 index 00000000..f1f8aa70 --- /dev/null +++ b/rest_VariantValidator/endpoints/variantvalidator_endpoints.py @@ -0,0 +1,263 @@ +# Import modules +from flask_restx import Namespace, Resource +from rest_VariantValidator.utils import exceptions, request_parser, representations, input_formatting, request_parser_g2t +from rest_VariantValidator.utils.object_pool import vval_object_pool, g2t_object_pool +from rest_VariantValidator.utils.limiter import limiter, vval_pool_limit, g2t_pool_limit +# get login authentication, if needed, or dummy auth if not present +try: + from VariantValidator_APIs.db_auth.verify_password import auth +except ModuleNotFoundError: + from rest_VariantValidator.utils.verify_password import auth + +# Create parser objects locally +parser = request_parser.parser +parser_g2t = request_parser_g2t.parser + +api = Namespace('VariantValidator', description='VariantValidator API Endpoints') + + +@api.route("/variantvalidator///", + strict_slashes=False) +@api.doc(description="This endpoint uses a dynamic rate limit based on pool availability.") +@api.param("select_transcripts", "***Return all possible transcripts***\n" + "\n***Return only 'select' transcripts***\n" + "> select\n" + "> mane_select\n" + "> mane (MANE and MANE Plus Clinical)\n" + "> refseq_select\n" + "\n***Single***\n" + "> NM_000093.4\n" + "\n***Multiple***\n" + "> NM_000093.4|NM_001278074.1|NM_000093.3") +@api.param("variant_description", "***HGVS***\n" + "> - NM_000088.3:c.589G>T\n" + "> - NC_000017.10:g.48275363C>A\n" + "> - NG_007400.1:g.8638G>T\n" + "> - LRG_1:g.8638G>T\n" + "> - LRG_1t1:c.589G>T\n" + "\n***Pseudo-VCF***\n" + "> - 17-50198002-C-A\n" + "> - 17:50198002:C:A\n" + "> - GRCh38-17-50198002-C-A\n" + "> - GRCh38:17:50198002:C:A\n" + "\n***Hybrid***\n" + "> - chr17:50198002C>A\n " + "> - chr17:50198002C>A(GRCh38)\n" + "> - chr17(GRCh38):50198002C>A\n" + "> - chr17:g.50198002C>A\n" + "> - chr17:g.50198002C>A(GRCh38)\n" + "> - chr17(GRCh38):g.50198002C>A") +@api.param("genome_build", "***Accepted:***\n" + "> - GRCh37\n" + "> - GRCh38\n" + "> - hg19\n" + "> - hg38") +class VariantValidatorClass(Resource): + @api.expect(parser, validate=True) + @auth.login_required() + @limiter.limit(vval_pool_limit) + def get(self, genome_build, variant_description, select_transcripts, user_id=None): + + # Get object from vval pool + vval = vval_object_pool.get_object() + + transcript_model = "refseq" + + # Deprecated check for 'all' or 'raw' for genomic variants + if ("all" in select_transcripts or "raw" in select_transcripts) and "auth" not in select_transcripts: + if not any(x in variant_description for x in ("c.", "n.", "r.", "p.")): + return {"Not Found": "Setting select_transcripts to 'all' or 'raw' is deprecated for genomic " + "variant processing using this endpoint. Contact admin on " + "https://variantvalidator.org/help/contact/ for updated instructions and" + " fair usage information; use another option; or use the LOVD endpoint which is " + "designed for integration into pipelines"}, 404 + elif "auth_all" in select_transcripts: + select_transcripts = "all" + elif "auth_raw" in select_transcripts: + select_transcripts = "raw" + + variant_description = input_formatting.format_input(variant_description) + select_transcripts = input_formatting.format_input(select_transcripts) + if select_transcripts == '["all"]': + select_transcripts = "all" + if select_transcripts == '["raw"]': + select_transcripts = "raw" + if select_transcripts == '["mane_select"]': + select_transcripts = "mane_select" + if select_transcripts == '["mane"]': + select_transcripts = "mane" + + try: + validate = vval.validate(variant_description, genome_build, select_transcripts, + transcript_set=transcript_model, lovd_syntax_check=True) + content = validate.format_as_dict(with_meta=True) + except Exception as e: + return {"error": str(e)}, 500 + finally: + vval_object_pool.return_object(vval) + + args = parser.parse_args() + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(content, 200, None) + else: + return content + + +@api.route("/variantvalidator_ensembl///", + strict_slashes=False) +class VariantValidatorEnsemblClass(Resource): + @api.expect(parser, validate=True) + @auth.login_required() + @limiter.limit(vval_pool_limit) + def get(self, genome_build, variant_description, select_transcripts, user_id=None): + vval = vval_object_pool.get_object() + transcript_model = "ensembl" + + if ("all" in select_transcripts or "raw" in select_transcripts) and "auth" not in select_transcripts: + if not any(x in variant_description for x in ("c.", "n.", "r.", "p.")): + return {"Not Found": "Setting select_transcripts to 'all' or 'raw' is deprecated for genomic " + "variant processing using this endpoint. Contact admin on " + "https://variantvalidator.org/help/contact/ for updated instructions and" + " fair usage information; use another option; or use the LOVD endpoint which is " + "designed for integration into pipelines"}, 404 + elif "auth_all" in select_transcripts: + select_transcripts = "all" + elif "auth_raw" in select_transcripts: + select_transcripts = "raw" + + variant_description = input_formatting.format_input(variant_description) + select_transcripts = input_formatting.format_input(select_transcripts) + if select_transcripts == '["all"]': + select_transcripts = "all" + if select_transcripts == '["raw"]': + select_transcripts = "raw" + if select_transcripts == '["mane_select"]': + select_transcripts = "mane_select" + if select_transcripts == '["mane"]': + select_transcripts = "mane" + + try: + validate = vval.validate(variant_description, genome_build, select_transcripts, + transcript_set=transcript_model, lovd_syntax_check=True) + content = validate.format_as_dict(with_meta=True) + except Exception as e: + return {"error": str(e)}, 500 + finally: + vval_object_pool.return_object(vval) + + args = parser.parse_args() + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(content, 200, None) + else: + return content + + +@api.route("/tools/gene2transcripts/", strict_slashes=False) +class Gene2transcriptsClass(Resource): + @api.expect(parser, validate=True) + @auth.login_required() + @limiter.limit(g2t_pool_limit) + def get(self, gene_query, user_id=None): + vval = g2t_object_pool.get_object() + gene_query = input_formatting.format_input(gene_query) + try: + content = vval.gene2transcripts(gene_query, lovd_syntax_check=True)[0] + except ConnectionError: + g2t_object_pool.return_object(vval) + raise exceptions.RemoteConnectionError("Cannot connect to rest.genenames.org, please try again later") + finally: + g2t_object_pool.return_object(vval) + + args = parser.parse_args() + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(content, 200, None) + else: + return content + + +@api.route("/tools/gene2transcripts_v2////" + "", strict_slashes=False) +class Gene2transcriptsV2Class(Resource): + @api.expect(parser_g2t, validate=True) + @auth.login_required() + @limiter.limit(g2t_pool_limit) + def get(self, gene_query, limit_transcripts, transcript_set, genome_build, user_id=None): + vval = g2t_object_pool.get_object() + + args = parser_g2t.parse_args() + bypass_genomic_spans = not bool(args['show_exon_info']) + + gene_query = input_formatting.format_input(gene_query) + limit_transcripts = input_formatting.format_input(limit_transcripts) + if len(limit_transcripts) == 1: + limit_transcripts = limit_transcripts[0] + + try: + if genome_build not in ["GRCh37", "GRCh38"]: + genome_build = None + if limit_transcripts in ["False", "false", False]: + limit_transcripts = None + content = vval.gene2transcripts(gene_query, select_transcripts=limit_transcripts, + transcript_set=transcript_set, genome_build=genome_build, + batch_output=True, validator=vval, + bypass_genomic_spans=bypass_genomic_spans, lovd_syntax_check=True) + except ConnectionError: + g2t_object_pool.return_object(vval) + raise exceptions.RemoteConnectionError("Cannot connect to rest.genenames.org, please try again later") + finally: + g2t_object_pool.return_object(vval) + + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(content, 200, None) + else: + return content + + +@api.route("/tools/hgvs2reference/", strict_slashes=False) +class Hgvs2referenceClass(Resource): + @api.expect(parser, validate=True) + @auth.login_required() + @limiter.limit(vval_pool_limit) + def get(self, hgvs_description, user_id=None): + vval = vval_object_pool.get_object() + try: + content = vval.hgvs2ref(hgvs_description) + except Exception as e: + return {"error": str(e)}, 500 + finally: + vval_object_pool.return_object(vval) + + args = parser.parse_args() + if args['content-type'] == 'application/json': + return representations.application_json(content, 200, None) + elif args['content-type'] == 'text/xml': + return representations.xml(content, 200, None) + else: + return content + + + +# +# Copyright (C) 2016-2021 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/__init__.py b/rest_VariantValidator/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rest_VariantValidator/utils/exceptions.py b/rest_VariantValidator/utils/exceptions.py new file mode 100644 index 00000000..029679ac --- /dev/null +++ b/rest_VariantValidator/utils/exceptions.py @@ -0,0 +1,19 @@ +class RemoteConnectionError(Exception): + code = 504 + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/input_formatting.py b/rest_VariantValidator/utils/input_formatting.py new file mode 100644 index 00000000..e13a191f --- /dev/null +++ b/rest_VariantValidator/utils/input_formatting.py @@ -0,0 +1,41 @@ +import json + + +def format_input(data_string): + """ + Takes an input string. Tries to convert from JSON to list, otherwise converts a string into a list. + Then goes on to check for pipe delimited data and splits if necessary + The output is a JSON array + """ + data_string = str(data_string) + try: + data_list = json.loads(data_string) + except json.decoder.JSONDecodeError: + data_intermediate = data_string.replace("|gom", "&gom") + data_intermediate = data_intermediate.replace("|lom", "&lom") + pre_data_list = data_intermediate.split("|") + if not isinstance(pre_data_list, list): + pre_data_list = [pre_data_list] + data_list = [] + for entry in pre_data_list: + entry = entry.replace("&", "|") + data_list.append(entry) + + return json.dumps(data_list) + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/limiter.py b/rest_VariantValidator/utils/limiter.py new file mode 100644 index 00000000..6f043ba0 --- /dev/null +++ b/rest_VariantValidator/utils/limiter.py @@ -0,0 +1,41 @@ +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address +from rest_VariantValidator.utils.object_pool import ( + vval_object_pool, + g2t_object_pool, + simple_variant_formatter_pool +) + +# Initialize Flask-Limiter +limiter = Limiter(key_func=get_remote_address) + +# -------------------------- +# Dynamic pool-based rate limiter +# -------------------------- +def pool_dynamic_limit(pool, max_rate, min_rate=1): + """ + Returns a dynamic Flask-Limiter rate string based on the number of available objects in a pool. + """ + with pool.lock: + if hasattr(pool, "objects"): # ObjectPool (Validator) + available = len(pool.objects) + elif hasattr(pool, "pool"): # SimpleVariantFormatterPool + available = len(pool.pool) + else: + available = max_rate + + # Clamp the rate between min_rate and max_rate + rate = max(min_rate, min(available, max_rate)) + return f"{rate}/second" + +# -------------------------- +# Convenience functions for each pool with custom max_rate +# -------------------------- +def vval_pool_limit(): + return pool_dynamic_limit(vval_object_pool, max_rate=8) + +def g2t_pool_limit(): + return pool_dynamic_limit(g2t_object_pool, max_rate=6) + +def formatter_pool_limit(): + return pool_dynamic_limit(simple_variant_formatter_pool, max_rate=8) diff --git a/rest_VariantValidator/utils/object_pool.py b/rest_VariantValidator/utils/object_pool.py new file mode 100644 index 00000000..d9fa8a50 --- /dev/null +++ b/rest_VariantValidator/utils/object_pool.py @@ -0,0 +1,70 @@ +import threading +from VariantValidator import Validator +from VariantFormatter import simpleVariantFormatter + + +class ObjectPool: + def __init__(self, object_type, initial_pool_size=10, max_pool_size=10): + self.pool_size = initial_pool_size + self.max_pool_size = max_pool_size + self.objects = [object_type() for _ in range(initial_pool_size)] + self.lock = threading.Lock() + self.condition = threading.Condition(self.lock) + + def get_object(self): + with self.condition: + while not self.objects: + # Wait until an object becomes available + self.condition.wait() + return self.objects.pop() + + def return_object(self, obj): + with self.condition: + if len(self.objects) < self.max_pool_size: + self.objects.append(obj) + self.condition.notify() # Notify waiting threads that an object is available + + +class SimpleVariantFormatterPool: + def __init__(self, initial_pool_size=10, max_pool_size=10): + self.pool_size = initial_pool_size + self.max_pool_size = max_pool_size + self.pool = [simpleVariantFormatter for _ in range(initial_pool_size)] + self.lock = threading.Lock() + self.condition = threading.Condition(self.lock) + + def get(self): + with self.condition: + while not self.pool: + # Wait until a formatter becomes available + self.condition.wait() + return self.pool.pop() + + def return_object(self, obj): + with self.condition: + if len(self.pool) < self.max_pool_size: + self.pool.append(obj) + self.condition.notify() # Notify waiting threads that a formatter is available + + +# Create shared object pools +vval_object_pool = ObjectPool(Validator, initial_pool_size=8, max_pool_size=10) +g2t_object_pool = ObjectPool(Validator, initial_pool_size=6, max_pool_size=10) +simple_variant_formatter_pool = SimpleVariantFormatterPool(initial_pool_size=8, max_pool_size=10) + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/representations.py b/rest_VariantValidator/utils/representations.py new file mode 100644 index 00000000..30f907ed --- /dev/null +++ b/rest_VariantValidator/utils/representations.py @@ -0,0 +1,43 @@ +# Import modules +from flask import make_response, jsonify +from dicttoxml import dicttoxml + +""" +Representations + - Adds a response-type into the "Response content type" drop-down menu displayed in Swagger + - When selected, the APP will return the correct response-header and content type + - The default for flask-RESTPlus is application/json + +Note + - These will only be used by namespaces so are undecorated. Decorated versions will appear in app.py +""" + + +def xml(data, code, headers): + data = dicttoxml(data) + resp = make_response(data, code) + resp.headers['Content-Type'] = 'text/xml' + return resp + + +def application_json(data, code, headers): + resp = make_response(jsonify(data), code) + resp.headers['Content-Type'] = 'application/json' + return resp + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/request_parser.py b/rest_VariantValidator/utils/request_parser.py new file mode 100644 index 00000000..22a63073 --- /dev/null +++ b/rest_VariantValidator/utils/request_parser.py @@ -0,0 +1,28 @@ +from flask_restx import reqparse + + +# Create a RequestParser object to identify specific content-type requests in HTTP URLs +# The request-parser allows us to specify arguments passed via a URL, in this case, ....?content-type=application/json +parser = reqparse.RequestParser() +parser.add_argument('content-type', + type=str, + help='***Select the response format***', + choices=['application/json', 'text/xml']) + + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/request_parser_g2t.py b/rest_VariantValidator/utils/request_parser_g2t.py new file mode 100644 index 00000000..c1870c23 --- /dev/null +++ b/rest_VariantValidator/utils/request_parser_g2t.py @@ -0,0 +1,41 @@ +from flask_restx import reqparse + +# Custom boolean conversion function +def str_to_bool(value): + if isinstance(value, bool): + return value + if value.lower() in ['true', '1', 't', 'yes', 'y']: + return True + elif value.lower() in ['false', '0', 'f', 'no', 'n']: + return False + else: + raise ValueError('Boolean value expected.') + +# Create a RequestParser object to identify specific content-type requests in HTTP URLs +parser = reqparse.RequestParser() +parser.add_argument('content-type', + type=str, + help='***Select the response format***', + choices=['application/json', 'text/xml']) +parser.add_argument('show_exon_info', + type=str_to_bool, + help='***Show Exon structures and alignment data***', + choices=[True, False]) + + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_VariantValidator/utils/verify_password.py b/rest_VariantValidator/utils/verify_password.py new file mode 100644 index 00000000..1e1d317c --- /dev/null +++ b/rest_VariantValidator/utils/verify_password.py @@ -0,0 +1,33 @@ +from functools import wraps + + +class DummyAuth: + # Unified verification decorator, dummy for non auth versions of the api + def login_required(self,null=None): + def login(f): + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + return login + + +auth = DummyAuth() + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + diff --git a/rest_VariantValidator/wsgi.py b/rest_VariantValidator/wsgi.py new file mode 100644 index 00000000..777ba473 --- /dev/null +++ b/rest_VariantValidator/wsgi.py @@ -0,0 +1,39 @@ +""" +Gunicorn wsgi gateway file +""" +import os +from rest_VariantValidator.app import application as app +from configparser import ConfigParser +from VariantValidator.settings import CONFIG_DIR + +config = ConfigParser() +config.read(CONFIG_DIR) + +if config["logging"]["log"] == "True": + app.debug = True + app.config['PROPAGATE_EXCEPTIONS'] = True +else: + app.debug = False + app.config['PROPAGATE_EXCEPTIONS'] = False + +if __name__ == '__main__': + # Read the port from the environment variable, defaulting to 8000 if not set + port = int(os.environ.get('PORT', 8000)) + app.run(host="127.0.0.1", port=port) + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/rest_cors_test.html b/rest_cors_test.html new file mode 100644 index 00000000..5f509862 --- /dev/null +++ b/rest_cors_test.html @@ -0,0 +1,39 @@ + + + + Access VariantValidator API + + +

Test of VV API for access errors

+

View the output from this page in the console. The console is opened by typing Ctrl-Shift-j.

+ +
+ + + diff --git a/rest_variantValidator/app.py b/rest_variantValidator/app.py deleted file mode 100644 index 41c5ed53..00000000 --- a/rest_variantValidator/app.py +++ /dev/null @@ -1,312 +0,0 @@ -# This application uses the flask restful API framework -import os -import re -import sys -from datetime import date, datetime, timedelta -import warnings - -# IMPORT FLASK MODULES -from flask import Flask ,request, jsonify, abort, url_for, g, send_file, redirect, Blueprint #, session, g, redirect, , abort, render_template, flash, make_response, abort -from flask_restful import Resource, Api, reqparse, abort, fields, marshal_with -from vv_flask_restful_swagger import swagger -from flask_log import Logging -from flask_mail import Mail, Message - -# Import variant validator code -import VariantValidator -vval = VariantValidator.Validator() - -# Import variantFormatter -import VariantFormatter -import VariantFormatter.simpleVariantFormatter - -# Extract API related metadata -config_dict = vval.my_config() -api_version = config_dict['variantvalidator_version'] -vf_api_version = VariantFormatter.__version__ - -# CREATE APP -application = Flask(__name__) - -# configure -application.config.from_object(__name__) - -# Wrap the Api with swagger.docs. -BaseURL = os.environ.get('SERVER_NAME') -if BaseURL is not None: - api = swagger.docs(Api(application), apiVersion=str(api_version), - basePath=BaseURL, - resourcePath='/', - produces=["application/json"], - api_spec_url='/webservices/variantvalidator', - description='VariantValidator web API' - ) -else: - api = swagger.docs(Api(application), apiVersion=str(api_version), - resourcePath='/', - produces=["application/json"], - api_spec_url='/webservices/variantvalidator', - description='VariantValidator web API' - ) - -# Create Logging instance -flask_log = Logging(application) -# Create Mail instance -mail = Mail(application) - - -# Resources -############ -""" -Essentially web pages that display json data -""" - -""" -Home -""" - -class home(Resource): - def get(self): - root_url = str(request.url_root) - full_url = root_url + 'webservices/variantvalidator.html' - return redirect(full_url) - - -""" -Simple interface for VariantValidator -""" -class variantValidator(Resource): - @swagger.operation( - notes='Submit a sequence variation to VariantValidator', - nickname='VariantValidator', - parameters=[ - { - "name": "genome_build", - "description": "Possible values: GRCh37, GRCh38, hg19, hg38", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - }, - { - "name": "variant_description", - "description": "Supported variant types: HGVS e.g. NM_000088.3:c.589G>T, NC_000017.10:g.48275363C>A, NG_007400.1:g.8638G>T, LRG_1:g.8638G>T, LRG_1t1:c.589G>T; pseudo-VCF e.g. 17-50198002-C-A, 17:50198002:C:A, GRCh3817-50198002-C-A, GRCh38:17:50198002:C:A; hybrid e.g. chr17:50198002C>A, chr17:50198002C>A(GRCh38), chr17:g.50198002C>A, chr17:g.50198002C>A(GRCh38)", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - }, - { - "name": "select_transcripts", - "description": "Possible values: all = return data for all relevant transcripts; single transcript id e.g. NM_000093.4; multiple transcript ids e.g. NM_000093.4|NM_001278074.1|NM_000093.3", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - } - ]) - - def get(self, genome_build, variant_description, select_transcripts): - try: - validate = vval.validate(variant_description, genome_build, select_transcripts) - validation = validate.format_as_dict(with_meta=True) - except Exception as e: - import traceback - import time - exc_type, exc_value, last_traceback = sys.exc_info() - te = traceback.format_exc() - tbk = str(exc_type) + str(exc_value) + '\n\n' + str(te) + '\n\nVariant = ' + str(variant_description) + ' and selected_assembly = ' + str(genome_build) + '/n' - error = str(tbk) - # Email admin - msg = Message(recipients=["variantvalidator@gmail.com"], - sender='apiValidator', - body=error + '\n\n' + time.ctime(), - subject='Major error recorded') - # Send the email - mail.send(msg) - error = {'flag' : ' Major error', - 'validation_error': 'A major validation error has occurred. Admin have been made aware of the issue'} - return error, 200, {'Access-Control-Allow-Origin': '*'} - - # Look for warnings - for key, val in validation.items(): - if key == 'flag' or key == 'metadata': - if key == 'flag' and str(val) == 'None': - import time - variant = variant_description - error = 'Variant = ' + str(variant_description) + ' and selected_assembly = ' + str(genome_build) + '\n' - # Email admin - msg = Message(recipients=["variantvalidator@gmail.com"], - sender='apiValidator', - body=error + '\n\n' + time.ctime(), - subject='Validation server error recorded') - # Send the email - mail.send(msg) - else: - continue - try: - if val['validation_warnings'] == 'Validation error': - import time - variant = variant_description - error = 'Variant = ' + str(variant_description) + ' and selected_assembly = ' + str(genome_build) + '\n' - # Email admin - msg = Message(recipients=["variantvalidator@gmail.com"], - sender='apiValidator', - body=error + '\n\n' + time.ctime(), - subject='Validation server error recorded') - # Send the email - mail.send(msg) - except TypeError: - pass - return validation, 200, {'Access-Control-Allow-Origin': '*'} - - - -""" -Return the transcripts for a gene -""" -class gene2transcripts(Resource): - @swagger.operation( - notes='Get a list of available transcripts for a gene by providing a valid HGNC gene symbol or transcript ID', - nickname='get genes2transcripts', - parameters=[ - { - "name": "gene_symbol", - "description": "HGNC gene symbol or transcript ID (Current supported transcript types: RefSeq)", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - } - ]) - def get(self, gene_symbol): - g2t = vval.gene2transcripts(gene_symbol) - return g2t, 200, {'Access-Control-Allow-Origin': '*'} - - -""" -Simple function that returns the reference bases for a hgvs description -""" -class hgvs2reference(Resource): - @swagger.operation( - notes='Get the reference bases for a HGVS variant description', - nickname='get reference bases', - parameters=[ - { - "name": "hgvs_description", - "description": "Sequence variation description in the HGVS format. Intronic descriptions in the context of transcript reference sequences are currently unsupported", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - } - ]) - def get(self, hgvs_description): - h2r = vval.hgvs2ref(hgvs_description) - # return jsonify(h2r) - return h2r, 200, {'Access-Control-Allow-Origin': '*'} - -""" -Simple interface for VariantFormatter -""" -class variantFormatter(Resource): - @swagger.operation( - notes='Submit a genomic sequence variant description to VariantFormatter', - nickname='VariantValidator', - parameters=[ - { - "name": "genome_build", - "description": "Possible values: GRCh37 or GRCh38", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - }, - { - "name": "variant_description", - "description": "Supported variant types: genomic HGVS e.g. NC_000017.10:g.48275363C>A, pseudo-VCF e.g. 17-50198002-C-A, 17:50198002:C:A (Note, for pVCF, multiple comma separated ALTs are supported). Multiple variants can be submitted, separated by the pipe '|' character. Recommended maximum is 10 variants per submission", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - }, - { - "name": "transcript_model", - "description": "Possible values: all = return data for all relevant transcripts; refseq = return data for RefSeq transcript models; refseq = return data for Ensembl transcript models:", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - }, - { - "name": "select_transcripts", - "description": "Possible values: None = return data for all relevant transcripts; single transcript id e.g. NM_000093.4; multiple transcript ids e.g. NM_000093.4|NM_001278074.1|NM_000093.3", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - }, - { - "name": "checkOnly", - "description": "Possible values: True or False. True will return ONLY the genomic variant descriptions and will not provide transcript and protein level descriptions", - "required": True, - "allowMultiple": False, - "dataType": 'string', - "paramType": "path" - } - ]) - def get(self, variant_description, genome_build, transcript_model=None, select_transcripts=None, checkOnly=False): - if transcript_model == 'None' or transcript_model == 'none': - transcript_model = None - if select_transcripts == 'None' or select_transcripts == 'none': - select_transcripts = None - if checkOnly == 'False' or checkOnly== 'false': - checkOnly = False - if checkOnly == 'True' or checkOnly== 'true': - checkOnly = True - v_form = VariantFormatter.simpleVariantFormatter.format(variant_description, genome_build, transcript_model, select_transcripts, checkOnly) - return v_form, 200, {'Access-Control-Allow-Origin': '*'} - - - -# ADD API resources to API handler - -# VariantValidator -api.add_resource(home, '/') -api.add_resource(variantValidator, '/variantvalidator///') -api.add_resource(gene2transcripts, '/tools/gene2transcripts/') -api.add_resource(hgvs2reference, '/tools/hgvs2reference/') - -# VariantFormatter -api.add_resource(variantFormatter, '/variantformatter/////') - - -if __name__ == '__main__': - from configparser import ConfigParser - from VariantValidator.settings import CONFIG_DIR - config = ConfigParser() - config.read(CONFIG_DIR) - if config["logging"]["log"] == "True": - application.debug = True - application.config['PROPAGATE_EXCEPTIONS'] = True - else: - application.debug = False - application.config['PROPAGATE_EXCEPTIONS'] = False - application.run(host="0.0.0.0", port=5000) - -# -# Copyright (C) 2019 VariantValidator Contributors -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 7c543656..00000000 --- a/setup.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# Prefer setuptools over distutils -from setuptools import setup, find_packages - -setup( - name='rest_VariantValidator', - version=open('VERSION.txt').read(), - description='Rest API for VariantValidator', - long_description=open('README.md').read(), - url='https://github.com/openvar/variantFormatter', - license="GNU AFFERO GENERAL PUBLIC LICENSE, Version 3 (https://www.gnu.org/licenses/agpl-3.0.en.html)", - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 3 - Alpha', - - # Audience - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Build Tools', - - # Specify the Python versions - 'Programming Language :: Python', - ], - - # What does your project relate to? - keywords=[ - "bioinformatics", - "computational biology", - "genome variants", - "genome variation", - "genomic variants", - "genomic variation", - "genomics", - "hgvs", - "HGVS", - "sequencevariants", - ], - - # List run-time dependencies here. These will be installed by pip when the project is installed. - install_requires=[ - "flask", - "flask-log", - "flask-mail", - "flask-cors", - "flask-restful", - "vv_flask_restful_swagger", - "VariantValidator", - "VariantFormatter", - "mysql-connector-python", - "flask-log", - "flask-mail", - "flask-restful", - "requests", - "pyliftover", - "vvhgvs", - ], - dependency_links=[ - "git+https://github.com/openvar/vv-flask-restful-swagger.git@master#egg=vv_flask_restful_swagger-0.20.1", - "git+https://github.com/openvar/variantValidator.git@develop_v1#egg=VariantValidator-1.0.0", - "git+https://github.com/openvar/variantFormatter.git@master#egg=VariantFormatter-1.0.0", - "git+https://github.com/openvar/vv_hgvs.git@master#egg=vvhgvs-1.0.0", - ], -) - -# -# Copyright (C) 2019 VariantValidator Contributors -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# diff --git a/swarms/__init__.py b/swarms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swarms/locust/__init__.py b/swarms/locust/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swarms/locust/locustfile.py b/swarms/locust/locustfile.py new file mode 100644 index 00000000..7a4eeef0 --- /dev/null +++ b/swarms/locust/locustfile.py @@ -0,0 +1,44 @@ +from swarms import HttpUser, TaskSet, task, between +import test_set + + +class UserBehavior(TaskSet): + @task(2) + def gene2transcripts_v2_task(self): + gene_symbol = test_set.gene_list() + url = f"/VariantValidator/tools/gene2transcripts_v2/{gene_symbol}/mane/all/GRCh38?content-type=application%2Fjson" + self.client.get(url) + + @task(1) + def variantvalidator_task(self): + variant_id = test_set.variant_list() + url = f"/VariantValidator/variantvalidator/GRCh37/{variant_id}/mane?content-type=application/json" + self.client.get(url) + + @task(3) + def additional_task(self): + odd_task = test_set.vf_list() + url = f"/LOVD/lovd/GRCh38/{odd_task}/all/all/False/True?content-type=application%2Fjson" + self.client.get(url) + + +class WebsiteUser(HttpUser): + tasks = [UserBehavior] + wait_time = between(0.1, 2) # seconds + +# +# Copyright (C) 2016-2025 VariantValidator Contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/swarms/locust/test_set.py b/swarms/locust/test_set.py new file mode 100644 index 00000000..f6c1499d --- /dev/null +++ b/swarms/locust/test_set.py @@ -0,0 +1,375 @@ +import random + + +def variant_list(): + tests = [ + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NM_000088.3:c.589GG>CT", + "NM_000094.3:c.6751-2_6751-3del", + "COL5A1:c.5071A>T", + "NG_007400.1:c.5071A>T", + "chr16:15832508_15832509delinsAC", + "chr16:15832508_15832509delinsAC", + "chr16:15832508_15832509delinsAC", + "chr16:15832508_15832509delinsAC", + "NM_000088.3:c.589-1GG>G", + "NM_000088.3:c.642+1GT>G", + "NM_000088.3:c.589-2AG>G", + "NC_000017.10:g.48279242G>T", + "NM_000500.7:c.-107-19C>T", + "NM_000518.4:c.-130C>T", + "NR_138595.1:n.-810C>T", + "NR_138595.1:n.1-810C>T", + "NC_000017.10:g.48261457_48261463TTATGTT=", + "NC_000017.10:g.48275363C>A", + "NM_000088.3:c.589-1G>T", + "NM_000088.3:c.591_593inv", + "11-5248232-T-A", + "NG_007400.1(NM_000088.3):c.589-1G>T", + "1:150550916G>A", + "1:150550916G>A", + "1:150550916G>A", + "1-150550916-G-A", + "1-150550916-G-A", + "1-150550916-G-A", + "NG_008123.1(LEPRE1_v003):c.2055+18G>A", + "NG_008123.1:c.2055+18G>A", + "NG_008123.1(NM_022356.3):c.2055+18G>A", + "NM_021983.4:c.490G>C", + "NM_032470.3:c.4del", + "NM_080680.2:c.1872+5G>A", + "NM_001194958.2:c.20C>A", + "NM_000022.2:c.534A>G", + "HSCHR6_MHC_SSTO_CTG1-3852542-C-G", + "NM_000368.4:c.363+1dupG", + "NM_000368.4:c.363dupG", + "NM_000089.3:c.1033_1035delGTT", + "NM_000089.3:c.1035_1035+2delTGT", + "NM_000088.3:c.2023_2028delGCAAGA", + "NM_000089.3:c.938-1delG", + "NM_000088.3:c.589G=", + "NM_000088.3:c.642A=", + "NM_000088.3:c.642+1GG>G", + "NM_000088.3:c.589-2GG>G", + "NM_000088.3:c.589-6_589-5insTTTT", + "NM_000088.3:c.642+3_642+4insAAAA", + "NM_000088.3:c.589-4_589-3insTT", + "NM_000088.3:c.589-8del", + "NM_000527.4:c.-187_-185delCTC", + "NM_206933.2:c.6317C>G", + "NC_000013.10:g.32929387T>C", + "NM_015102.3:c.2818-2T>A", + "19-41123094-G-GG", + "19-41123094-G-GG", + "19-41123094-G-GG", + "15-72105928-AC-A", + "15-72105928-AC-A", + "15-72105928-AC-A", + "15-72105928-AC-A", + "12-122064773-CCCGCCA-C", + "12-122064774-CCGCCA-CCGCCA", + "12-122064773-CCCGCCACCGCCACCGC-CCCGCCACCGCCGCCGTC", + "NC_000012.11:g.122064777C>A", + "NC_000012.11:g.122064776delG", + "NC_000012.11:g.122064776dupG", + "NC_000012.11:g.122064776_122064777insTTT", + "NC_000012.11:g.122064772_122064775del", + "NC_000012.11:g.122064772_122064775dup", + "NC_000012.11:g.122064773_122064774insTTTT", + "NC_000012.11:g.122064772_122064777del", + "NC_000012.11:g.122064772_122064777dup", + "NC_000012.11:g.122064779_122064782dup", + "NC_000012.11:g.122064772_122064782del", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000002.11:g.95847041_95847043GCG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000003.11:g.14561629_14561630GC=", + "NC_000003.11:g.14561629_14561630GC=", + "NC_000003.11:g.14561629_14561630insG", + "NC_000003.11:g.14561629_14561630insG", + "NC_000004.11:g.140811111_140811122del", + "NC_000004.11:g.140811111_140811122CTGCTGCTGCTG=", + "NC_000004.11:g.140811117_140811122del", + "NC_000004.11:g.140811111_140811117del", + "NC_000004.11:g.140811117C>A", + "NC_000002.11:g.73675227_73675228insCTC", + "9-136132908-T-TC", + "9-136132908-TAC-TCA", + "9-136132908-TA-TA", + "NM_020469.2:c.258delG", + "NM_020469.2:c.260_262TGA=", + "NM_020469.2:c.261delG", + "NM_020469.2:c.261dupG", + "NM_020469.2:c.261_262insTT", + "NC_000019.10:g.50378563_50378564insTAC", + "NC_000019.10:g.50378563_50378564insC", + "NC_000019.10:g.50378564_50378565insTACA", + "NC_000019.10:g.50378565_50378567dup", + "NC_000019.10:g.50378563_50378564=", + "NC_000019.10:g.50378563_50378564insTCGG", + "NC_000019.10:g.50378563_50378564insC", + "NC_000019.10:g.50378563delinsTTAC", + "NC_000019.10:g.50378563_50378564insTAAC", + "NC_000019.10:g.50378562_50378565del", + "NC_000019.10:g.50378562_50378565delinsTC", + "NC_000007.14:g.149779575_149779577delinsT", + "NC_000007.14:g.149779575_149779577=", + "NC_000007.14:g.149779576_149779578del", + "NC_000007.14:g.149779577del", + "NC_000007.14:g.149779573_149779579del", + "NC_000007.14:g.149779573_149779579delinsCA", + "NC_000004.12:g.139889957_139889968del", + "NM_015120.4:c.35T>C", + "NM_015120.4:c.39G>C", + "NM_015120.4:c.34C>T", + "NC_000002.11:g.73613030C>T", + "NM_000088.3:c.590_591inv", + "NM_024989.3:c.1778_1779inv", + "NM_032815.3:c.555_556inv", + "NM_001034853.1:c.2847_2848delAGinsCT", + "NM_000038.5:c.3927_3928delAAinsTT", + "NM_198180.2:c.408_410delGTG", + "NM_080877.2:c.1733_1735delinsTTT", + "NM_080877.2:c.1735_1737delinsTGA", + "NM_080877.2:c.1735_1737delinsTAATTGTTC", + "NM_080877.2:c.1737delinsATTGTTC", + "NM_006138.4:c.3_4inv", + "NM_000088.3:c.4392_*2inv", + "NM_000088.3:c.4392_*5inv", + "NM_000088.3:c.4390_*7inv", + "NM_000088.3:c.4392_*2delinsAGAG", + "NM_000088.3:c.589_591delinsAGAAGC", + "NM_000885.5:c.*2536delinsAGAAAAATCA", + "NM_002693.2:c.-186_-185delinsCC", + "NG_009616.1:g.29052_29053insCTACATAG", + "NM_000061.2:c.588_588+1insCTACATAG", + "NM_000061.2:c.588_589insCTACATAG", + "NM_005732.3:c.2923-5insT", + "NM_198283.1(EYS):c.*743120C>T", + "NM_133379.4(TTN):c.*265+26591C>T", + "NM_000088.3:c.589-2_589-1AG>G", + "NM_000088.3:c.642+1_642+2delGTinsG", + "NM_001260.1:c.967C>T", + "NM_004415.3:c.1-1insA", + "NM_000273.2:c.1-5028_253del", + "NM_002929.2:c.1006C>T", + "NR_125367.1:n.167+18165G>A", + "NM_006005.3:c.3071_3073delinsTTA", + "NM_000089.3:n.1504_1506del", + "NC_012920.1:m.1011C>T", + "NC_000006.11:g.90403795G=", + "NC_000006.11:g.90403795G=", + "1-169519049-T-.", + "1-169519049-T-.", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NM_000251.1:c.1296_1348del", + "NM_000088.3:c.2023_2028del", + "NM_000088.3:c.2024_2028+1del", + "ENST00000450616.1:n.31+1G>C", + "ENST00000491747:c.5071A>T", + "NM_000088.3:c.589G>T", + "NG_007400.1:g.8638G>T", + "LRG_1:g.8638G>T", + "LRG_1t1:c.589G>T", + "chr16:15832508_15832509delinsAC", + "chr16:15832508_15832509delinsAC", + "chr16:15832508_15832509delinsAC", + "chr16:g.15832508_15832509delinsAC", + "NG_012386.1:g.24048dupG", + "NG_012386.1:g.24048dupG", + "NG_012386.1:g.24048dupG", + "NM_033517.1:c.1307_1309delCGA", + "HG1311_PATCH-33720-CCGA-C", + "2-73675227-TCTC-TCTCCTC", + "2-73675227-TC-TC", + "3-14561627-AG-AGG", + "3-14561627-AG-AGG", + "3-14561630-CC-CC", + "3-14561630-CC-CC", + "6-90403795-G-G", + "6-90403795-G-G", + "6-90403795-G-A", + "6-90403795-G-A", + "6-32012992-CG-C", + "6-32012992-CG-C", + "17-48275363-C-A", + "17-48275364-C-A", + "17-48275359-GGA-TCC", + "7-94039128-CTTG-C", + "9-135800972-AC-ACC", + "9-135800972-AC-ACC", + "9-135800972-AC-ACC", + "1-43212925-C-T", + "1-43212925-C-T", + "1-43212925-C-T", + "HG987_PATCH-355171-C-A", + "20-43252915-T-C", + "20-43252915-T-C", + "20-43252915-T-C", + "20-43252915-T-C", + "20-43252915-T-C", + "1-216219781-A-C", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "2-209113113-G-A,C,T", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000005.9:g.35058665_35058666CA=", + "NC_000002.11:g.73675227_73675229delTCTinsTCTCTC", + "NC_000002.11:g.73675227_73675228insCTC", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NC_000017.10:g.5286863_5286889AGTGTTTGGAATTTTCTGTTCATATAG=", + "NM_000828.4:c.-2dupG", + "X-122318386-A-AGG", + "X-122318386-A-AGG", + "X-122318386-A-AGG", + "NM_000828.4:c.-2G>T", + "NM_000828.4:c.-2G=", + "X-122318386-A-AT", + "X-122318386-A-AT", + "X-122318386-A-AT", + "NM_000828.4:c.-2_-1insT", + "NM_000828.4:c.-3_-2insT", + "NM_000828.4:c.-2delGinsTT", + "NM_000828.4:c.-2_-1delGCinsTT", + "NM_000828.4:c.-3_-2delAGinsTT", + "15-72105929-C-C", + "15-72105929-C-C", + "15-72105929-C-C", + "15-72105929-C-C", + "15-72105928-AC-ATT", + "15-72105928-AC-ATT", + "15-72105928-AC-ATT", + "15-72105928-AC-ATT", + "15-72105928-ACC-ATT", + "15-72105928-ACC-ATT", + "15-72105928-ACC-ATT", + "15-72105928-ACC-ATT", + "15-72105927-GACC-GTT", + "15-72105927-GACC-GTT", + "15-72105927-GACC-GTT", + "15-72105927-GACC-GTT", + "19-41123093-A-AG", + "19-41123093-A-AG", + "19-41123093-A-AG", + "19-41123093-A-AT", + "19-41123093-A-AT", + "19-41123093-A-AT", + "19-41123093-AG-A", + "19-41123093-AG-A", + "19-41123093-AG-A", + "19-41123093-AG-AG", + "19-41123093-AG-AG", + "19-41123093-AG-AG", + "NC_000003.11:g.14561629_14561630insG", + "NC_000003.11:g.14561629_14561630insG", + "NM_012309.4:c.913-5058G>A", + "LRG_199t1:c.2376[G>C];[G>C]", + "LRG_199t1:c.2376[G>C];[G>C]", + "LRG_199t1:c.[2376G>C];[3103del]", + "LRG_199t1:c.[2376G>C];[3103del]", + "LRG_199t1:c.[4358_4359del;4361_4372del]", + "LRG_199t1:c.2376G>C(;)3103del", + "LRG_199t1:c.2376G>C(;)3103del", + "LRG_199t1:c.2376[G>C];[(G>C)]", + "LRG_199t1:c.[2376G>C];[?]", + "LRG_199t1:c.[296T>G;476T=];[476T=](;)1083A>C", + "LRG_199t1:c.[296T>G;476T=];[476T=](;)1083A>C", + "LRG_199t1:c.[296T>G;476T=];[476T=](;)1083A>C", + "LRG_199t1:c.[296T>G];[476T>C](;)1083A>C(;)1406del", + "LRG_199t1:c.[296T>G];[476T>C](;)1083A>C(;)1406del", + "LRG_199t1:c.[296T>G];[476T>C](;)1083A>C(;)1406del", + "LRG_199t1:c.[296T>G];[476T>C](;)1083A>C(;)1406del", + "LRG_199t1:c.[976-20T>A;976-17_976-1dup]" + ] + random_number = random.randint(0, 319) + odd_job = tests[random_number] + return odd_job + + +def gene_list(): + tests = [ + "BRCA1", + "COL1A1", + "TP53", + "COL5A1", + "BRCA2", + "HBB", + "DMD", + "BRAF", + "HTT", + "NR2E3" + ] + + random_number = random.randint(0, 6) + random_task = tests[random_number] + return random_task + + +def vf_list(): + tests = [ + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NC_000016.9:g.2099572TC>T", + "NM_000088.3:c.589GG>CT", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=", + "NC_000005.9:g.35058667_35058668AG=" + ] + random_number = random.randint(0, len(tests) - 1) + odd_task = tests[random_number] + return odd_task diff --git a/swarms/tests/__init__.py b/swarms/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swarms/tests/test_curl_restvv_lovd_endpoint.py b/swarms/tests/test_curl_restvv_lovd_endpoint.py new file mode 100644 index 00000000..2993908b --- /dev/null +++ b/swarms/tests/test_curl_restvv_lovd_endpoint.py @@ -0,0 +1,163 @@ +import json +import subprocess +import time +import os + +# Updated base URL +BASE_URL = "https://www183.lamp.le.ac.uk/LOVD/lovd" +THROTTLE_SECONDS = 0.3 # ~3 requests/sec (API allows 4/sec) + +def run_curl( + variant, + genome_build="GRCh38", + transcript_model="refseq", + select_transcripts="None", + liftover="False", + checkonly="False"): + """ + Run curl against the LOVD VariantValidator API and return parsed JSON. + Automatically includes Authorization header if RESTVV_BEARER_TOKEN is set. + """ + time.sleep(THROTTLE_SECONDS) # avoid rate limiting + + url = ( + f"{BASE_URL}/{genome_build}/{variant}/" + f"{transcript_model}/{select_transcripts}/" + f"{liftover}/{checkonly}" + f"?content-type=application%2Fjson" + ) + + # Get bearer token from environment (set by your token script) + bearer_token = os.getenv("RESTVV_BEARER_TOKEN") + + # Build curl command + cmd = [ + "curl", "-s", "-X", "GET", url, + "-H", "accept: application/json", + "-H", "Content-Type: application/json" + ] + + # Add Authorization header if token exists + if bearer_token: + cmd += ["-H", f"Authorization: Bearer {bearer_token}"] + + # Run curl and parse output + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return json.loads(result.stdout) + + +# ------------------------------------------------------------------------- +# All your existing test classes stay exactly as they were +# ------------------------------------------------------------------------- + +class TestVariantInputs: + def test_hybrid_syntax_1(self): + data = run_curl("chr17:50198002C>A") + entry = data["chr17:50198002C>A"]["NC_000017.11:g.50198002C>A"] + assert entry["g_hgvs"] == "NC_000017.11:g.50198002C>A" + + def test_hybrid_syntax_2(self): + data = run_curl("17:50198002C>A") + entry = data["17:50198002C>A"]["NC_000017.11:g.50198002C>A"] + assert entry["g_hgvs"] == "NC_000017.11:g.50198002C>A" + + +class TestTranscriptSelection: + def test_transcript_selection_raw(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="raw") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.3" in hgvs_t_and_p + assert "NM_005859.4" in hgvs_t_and_p + assert "NM_005859.5" in hgvs_t_and_p + + def test_transcript_selection_all(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="all") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.5" in hgvs_t_and_p + assert "NM_005859.3" not in hgvs_t_and_p + assert "NM_005859.4" not in hgvs_t_and_p + + def test_transcript_selection_mane_select(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="mane_select") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.5" in hgvs_t_and_p + + def test_transcript_selection_select(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="select") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.4" in hgvs_t_and_p + assert "NM_005859.5" in hgvs_t_and_p + assert "NM_005859.3" not in hgvs_t_and_p + + def test_transcript_selection_nm(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="NM_005859.4") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.4" in hgvs_t_and_p + assert "NM_005859.3" not in hgvs_t_and_p + assert "NM_005859.5" not in hgvs_t_and_p + + def test_transcript_selection_mane(self): + data = run_curl("NC_000007.14:g.140924703T>C", select_transcripts="mane") + hgvs_t_and_p = data["NC_000007.14:g.140924703T>C"]["NC_000007.14:g.140924703T>C"]["hgvs_t_and_p"] + assert "NM_004333.6" in hgvs_t_and_p + assert "NM_001374258.1" in hgvs_t_and_p + assert "NM_001354609.1" not in hgvs_t_and_p + + +class TestVariantAutoCases: + def test_variant1_bad_build(self): + v = "NC_000019.10:g.50378563_50378564insTAC" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert "chromosome ID NC_000019.10 is not associated" in entry["genomic_variant_error"] + + def test_variant2_mismatched_reference(self): + v = "11-5248232-A-T" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert "does not agree with reference sequence" in entry["genomic_variant_error"] + + def test_variant3_ref_mismatch(self): + v = "NC_000012.11:g.122064777A>C" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert "does not agree with reference sequence" in entry["genomic_variant_error"] + + def test_variant4_synonymous(self): + v = "NC_000002.11:g.73613030C>T" + data = run_curl(v, genome_build="GRCh37", select_transcripts="all") + entry = data[v][v] + assert entry["genomic_variant_error"] is None + assert "NM_015120.4" in entry["hgvs_t_and_p"] + + def test_variant5_x_chromosome(self): + v = "NC_000023.10:g.33229673A>T" + data = run_curl(v, genome_build="GRCh37", select_transcripts="raw") + entry = data[v][v] + assert entry["genomic_variant_error"] is None + assert "NM_000109.3" in entry["hgvs_t_and_p"] + + def test_variant6_intergenic(self): + v = "NC_000017.10:g.48279242G>T" + data = run_curl(v, genome_build="GRCh37", select_transcripts="all") + entry = data[v][v] + assert entry["hgvs_t_and_p"] == {"intergenic": {"alt_genomic_loci": None}} + + def test_variant7_identity(self): + v = "NC_000017.10:g.48261457_48261463TTATGTT=" + data = run_curl(v, genome_build="GRCh37", select_transcripts="raw") + entry = data[v][v] + assert "NM_000088.3" in entry["hgvs_t_and_p"] + + def test_variant8_missense(self): + v = "NC_000017.10:g.48275363C>A" + data = run_curl(v, genome_build="GRCh37", select_transcripts="raw") + entry = data[v][v] + assert "NM_000088.3" in entry["hgvs_t_and_p"] + + def test_variant9_roundtrip_consistency(self): + v = "11-5248232-T-A" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert entry["p_vcf"] == v + assert entry["genomic_variant_error"] is None diff --git a/swarms/tests/test_curl_terracipher_vfv2_endpoint.py b/swarms/tests/test_curl_terracipher_vfv2_endpoint.py new file mode 100644 index 00000000..31acd541 --- /dev/null +++ b/swarms/tests/test_curl_terracipher_vfv2_endpoint.py @@ -0,0 +1,181 @@ +import json +import subprocess +import time +import os + +# ----------------------------- +# API Base and Throttling +# ----------------------------- +BASE_URL = "https://shaipup.com/market/shaip?owner=uominnovationfactory&shaip=variantvalidatorapi" +THROTTLE_SECONDS = 0 # ~3 requests/sec (API allows 4/sec) + +# Tokens from environment +BEARER_TOKEN = os.getenv("SHAIP_BEARER_TOKEN") +COOKIE_TOKEN = os.getenv("SHAIP_COOKIE_TOKEN") + +if not BEARER_TOKEN: + raise RuntimeError("Please set SHAIP_BEARER_TOKEN in your environment") +if not COOKIE_TOKEN: + raise RuntimeError("Please set SHAIP_COOKIE_TOKEN in your environment") + + +def run_curl( + variant, + genome_build="GRCh38", + transcript_model="refseq", + select_transcripts="None", + liftover="False", + checkonly="False", +): + """ + Run curl against the TerraCipher VariantValidator API and return parsed JSON. + Uses POST with JSON body. + """ + time.sleep(THROTTLE_SECONDS) # avoid rate limiting + + # Build JSON payload + payload = [ + { + "select_transcripts": [select_transcripts], + "liftover": liftover.lower() == "true", + "checkonly": checkonly.lower() == "true", + "transcript_model": transcript_model, + "genome_build": genome_build, + "variant_description": [variant], + } + ] + + cmd = [ + "curl", + "--location", + "--request", "POST", BASE_URL, + "--header", f"Authorization: Bearer {BEARER_TOKEN}", + "--header", "Content-Type: application/json", + "--header", f"Cookie: {COOKIE_TOKEN}", + "--data-raw", json.dumps(payload), + "-s", # silent + ] + + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return json.loads(result.stdout) + +# ----------------------------- +# Variant Inputs +# ----------------------------- +class TestVariantInputs: + def test_hybrid_syntax_1(self): + data = run_curl("chr17:50198002C>A") + entry = data["chr17:50198002C>A"]["NC_000017.11:g.50198002C>A"] + assert entry["g_hgvs"] == "NC_000017.11:g.50198002C>A" + + def test_hybrid_syntax_2(self): + data = run_curl("17:50198002C>A") + entry = data["17:50198002C>A"]["NC_000017.11:g.50198002C>A"] + assert entry["g_hgvs"] == "NC_000017.11:g.50198002C>A" + + +# ----------------------------- +# Transcript Selection +# ----------------------------- +class TestTranscriptSelection: + def test_transcript_selection_raw(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="raw") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.3" in hgvs_t_and_p + assert "NM_005859.4" in hgvs_t_and_p + assert "NM_005859.5" in hgvs_t_and_p + + def test_transcript_selection_all(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="all") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.5" in hgvs_t_and_p + assert "NM_005859.3" not in hgvs_t_and_p + assert "NM_005859.4" not in hgvs_t_and_p + + def test_transcript_selection_mane_select(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="mane_select") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.5" in hgvs_t_and_p + + def test_transcript_selection_select(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="select") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.4" in hgvs_t_and_p + assert "NM_005859.5" in hgvs_t_and_p + assert "NM_005859.3" not in hgvs_t_and_p + + def test_transcript_selection_nm(self): + data = run_curl("NC_000005.10:g.140114829del", select_transcripts="NM_005859.4") + hgvs_t_and_p = data["NC_000005.10:g.140114829del"]["NC_000005.10:g.140114829del"]["hgvs_t_and_p"] + assert "NM_005859.4" in hgvs_t_and_p + assert "NM_005859.3" not in hgvs_t_and_p + assert "NM_005859.5" not in hgvs_t_and_p + + def test_transcript_selection_mane(self): + data = run_curl("NC_000007.14:g.140924703T>C", select_transcripts="mane") + hgvs_t_and_p = data["NC_000007.14:g.140924703T>C"]["NC_000007.14:g.140924703T>C"]["hgvs_t_and_p"] + assert "NM_004333.6" in hgvs_t_and_p + assert "NM_001374258.1" in hgvs_t_and_p + assert "NM_001354609.1" not in hgvs_t_and_p + + +# ----------------------------- +# Auto/Edge Case Variants +# ----------------------------- +class TestVariantAutoCases: + def test_variant1_bad_build(self): + v = "NC_000019.10:g.50378563_50378564insTAC" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert "chromosome ID NC_000019.10 is not associated" in entry["genomic_variant_error"] + + def test_variant2_mismatched_reference(self): + v = "11-5248232-A-T" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert "does not agree with reference sequence" in entry["genomic_variant_error"] + + def test_variant3_ref_mismatch(self): + v = "NC_000012.11:g.122064777A>C" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert "does not agree with reference sequence" in entry["genomic_variant_error"] + + def test_variant4_synonymous(self): + v = "NC_000002.11:g.73613030C>T" + data = run_curl(v, genome_build="GRCh37", select_transcripts="all") + entry = data[v][v] + assert entry["genomic_variant_error"] is None + assert "NM_015120.4" in entry["hgvs_t_and_p"] + + def test_variant5_x_chromosome(self): + v = "NC_000023.10:g.33229673A>T" + data = run_curl(v, genome_build="GRCh37", select_transcripts="raw") + entry = data[v][v] + assert entry["genomic_variant_error"] is None + assert "NM_000109.3" in entry["hgvs_t_and_p"] + + def test_variant6_intergenic(self): + v = "NC_000017.10:g.48279242G>T" + data = run_curl(v, genome_build="GRCh37", select_transcripts="all") + entry = data[v][v] + assert entry["hgvs_t_and_p"] == {"intergenic": {"alt_genomic_loci": None}} + + def test_variant7_identity(self): + v = "NC_000017.10:g.48261457_48261463TTATGTT=" + data = run_curl(v, genome_build="GRCh37", select_transcripts="raw") + entry = data[v][v] + assert "NM_000088.3" in entry["hgvs_t_and_p"] + + def test_variant8_missense(self): + v = "NC_000017.10:g.48275363C>A" + data = run_curl(v, genome_build="GRCh37", select_transcripts="raw") + entry = data[v][v] + assert "NM_000088.3" in entry["hgvs_t_and_p"] + + def test_variant9_roundtrip_consistency(self): + v = "11-5248232-T-A" + data = run_curl(v, genome_build="GRCh37") + entry = data[v][v] + assert entry["p_vcf"] == v + assert entry["genomic_variant_error"] is None diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py new file mode 100644 index 00000000..05a14f4b --- /dev/null +++ b/tests/test_endpoints.py @@ -0,0 +1,153 @@ +# Import necessary packages +import pytest +import time +from rest_VariantValidator.app import application # Import your Flask app + + +# Fixture to set up the test client +@pytest.fixture(scope='module') +def client(): + application.testing = True + test_client = application.test_client() # Create a test client to interact with the app + + yield test_client # This is where the tests will run + + +# Test function for the /hello/ endpoint +def test_hello_endpoint(client): + response = client.get('/hello/') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + + +def test_lovd_endpoint(client): + response = client.get('/LOVD/lovd/GRCh38/17-50198002-C-A/all/mane/True/False?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + assert "17-50198002-C-A" in response.json.keys() + # time.sleep(1) + +def test_lovd_endpoint_multi(client): + response = client.get('/LOVD/lovd/GRCh38/17-50198002-C-A|17-50198002-C-T/all/mane/True/False?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + assert "17-50198002-C-A" in response.json.keys() + assert "17-50198002-C-T" in response.json.keys() + # time.sleep(1) + + +def test_vf_endpoint(client): + response = client.get('/VariantFormatter/variantformatter/GRCh38/NC_000017.10%3Ag.48275363C%3EA/all/mane/True?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/NM_000088.3%3Ac.589G%3ET/all?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + assert "NM_000088.3:c.589G>T" in response.json.keys() + # time.sleep(1) + + +def test_vv_endpoint_multi(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/NM_000088.3%3Ac.589G%3ET|NM_000088.3%3Ac.589G%3EA/mane?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + assert "NM_000088.3:c.589G>T" in response.json.keys() + assert "NM_000088.3:c.589G>A" in response.json.keys() + # time.sleep(1) + + +def test_g2t_endpoint(client): + response = client.get('/VariantValidator/tools/gene2transcripts/COL1A1?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert len(response.json) == 8 + # time.sleep(1) + + +def test_g2t2_endpoint(client): + response = client.get('/VariantValidator/tools/gene2transcripts_v2/COL1A1%7CCOL1A2%7CCOL5A1/mane/all/GRCh38?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert len(response.json) == 3 + # time.sleep(1) + + +def test_h2ref_endpoint(client): + response = client.get('/VariantValidator/tools/hgvs2reference/NM_000088.3%3Ac.589G%3ET?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + # time.sleep(1) + + +def test_vv_endpoint_all_tx(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/NM_000088.3%3Ac.589G%3ET/all?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint_all_vcf(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/17-50198002-C-A/all?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 404 # Check if the response status code is 200 OK + assert "metadata" not in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint_all_genomic(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/NC_000017.10:g.48275363C>A/all?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 404 # Check if the response status code is 200 OK + assert "metadata" not in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint_mane_genomic(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/NC_000017.10:g.48275363C>A/mane?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint_mane_vcf(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/17-50198002-C-A/mane?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint_auth_raw_vcf(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/17-50198002-C-A/auth_all?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_endpoint_auth_all_vcf(client): + response = client.get('/VariantValidator/variantvalidator/GRCh38/17-50198002-C-A/auth_raw?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + # time.sleep(1) + + +def test_vv_g2tv2(client): + response = client.get('/VariantValidator/tools/gene2transcripts_v2/AMPD1/mane/refseq/GRCh38?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert len(response.json) == 1 + # time.sleep(1) + +def test_lovd_endpoint_all(client): + response = client.get('/LOVD/lovd/GRCh37/1%3A43815009%3AG%3AT/refseq/all/False/True?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + assert "1:43815009:G:T" in response.json.keys() + # time.sleep(1) + + +def test_lovd_endpoint_raw(client): + response = client.get('/LOVD/lovd/GRCh37/1%3A43815009%3AG%3AT/refseq/raw/False/True?content-type=application%2Fjson') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response + assert "1:43815009:G:T" in response.json.keys() + # time.sleep(1) diff --git a/tests/test_handlers.py b/tests/test_handlers.py new file mode 100644 index 00000000..4e4ae67c --- /dev/null +++ b/tests/test_handlers.py @@ -0,0 +1,44 @@ +# Import necessary packages +import pytest +from rest_VariantValidator.app import application # Import your Flask app + + +# Fixture to set up the test client +@pytest.fixture(scope='module') +def client(): + application.testing = False + application.debug = False + application.config['PROPAGATE_EXCEPTIONS'] = True + return application.test_client() # Create a test client to interact with the app + + +# Test function for exception handlers +def test_bad_request_url(client): + response = client.get('/nonexistent/', headers={'Content-Type': 'application/json'}) + assert response.status_code == 404 # Check if the response status code is 404 NOT FOUND + assert response.json["message"] == "Requested Endpoint not found: See the documentation at https://rest.variantvalidator.org" + + +def test_bad_request_code(client): + response = client.get('/trigger_error/404', headers={'Content-Type': 'application/json'}) + assert response.status_code == 404 # Check if the response status code is 404 NOT FOUND + assert response.json["message"] == "Requested Endpoint not found: See the documentation at https://rest.variantvalidator.org" + + +def test_error_code(client): + response = client.get('/hello/trigger_error/500', headers={'Content-Type': 'application/json'}) + assert response.status_code == 500 + assert response.json["message"] == "Unhandled error: contact https://variantvalidator.org/contact_admin/" + + +def test_limit_code(client): + response = client.get('/hello/trigger_error/429', headers={'Content-Type': 'application/json'}) + assert response.status_code == 429 + assert response.json["message"] == "Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org" + + +def test_connection_error(client): + response = client.get('/hello/trigger_error/999', headers={'Content-Type': 'application/json'}) # Send a GET request to a nonexistent endpoint + assert response.status_code == 504 # Check if the response status code is 404 NOT FOUND + assert response.json["message"] == "https://rest.variantvalidator.org/variantvalidator currently unavailable" + diff --git a/tests/test_input_formatting.py b/tests/test_input_formatting.py new file mode 100644 index 00000000..7b626e24 --- /dev/null +++ b/tests/test_input_formatting.py @@ -0,0 +1,55 @@ +import unittest +import json +from rest_VariantValidator.utils import input_formatting + + +class TestFormatInput(unittest.TestCase): + + def test_json_input(self): + data_string = '[1, 2, 3]' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '[1, 2, 3]') + + def test_string_input(self): + data_string = '1|2|3' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '["1", "2", "3"]') + + def test_json_array_input(self): + data_string = '["a", "b", "c"]' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '["a", "b", "c"]') + + def test_mixed_input(self): + data_string = '[1, 2, 3]|4|5' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '["[1, 2, 3]", "4", "5"]') + + def test_pipe_and_special_characters(self): + data_string = 'a|b|c|gomd|e|lomf' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '["a", "b", "c|gomd", "e|lomf"]') + + def test_empty_input(self): + data_string = '' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '[""]') + + def test_multiple_pipe_symbols(self): + data_string = 'a||b||c' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '["a", "", "b", "", "c"]') + + def test_json_array_string_input(self): + data_string = '["apple", "banana", "cherry"]' + result = input_formatting.format_input(data_string) + self.assertEqual(result, '["apple", "banana", "cherry"]') + + def test_result_is_json(self): + data_string = '[1, 2, 3]' + result = input_formatting.format_input(data_string) + self.assertIsInstance(result, str) + json_result = json.loads(result) + self.assertEqual(json_result, [1, 2, 3]) + + diff --git a/tests/test_internal_servers.py b/tests/test_internal_servers.py new file mode 100644 index 00000000..4c9131a1 --- /dev/null +++ b/tests/test_internal_servers.py @@ -0,0 +1,79 @@ +import os +import subprocess +import time +from unittest import TestCase +import requests + +class TestInternalServers(TestCase): + @classmethod + def setUpClass(cls): + # Set the PORT environment variable for WSGI server + os.environ['PORT'] = '8002' + + # Start the WSGI server as a separate process + cls.wsgi_server_process = subprocess.Popen(['python', 'rest_VariantValidator/wsgi.py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(60) + + # Check if the WSGI server started correctly + if cls.wsgi_server_process.poll() is not None: + stdout, stderr = cls.wsgi_server_process.communicate() + print(f"WSGI Server stdout: {stdout}") + print(f"WSGI Server stderr: {stderr}") + raise RuntimeError('WSGI server failed to start') + + # Set the PORT environment variable for APP server + os.environ['PORT'] = '5002' + + # Start the app server as a separate process + cls.app_server_process = subprocess.Popen(['python', 'rest_VariantValidator/app.py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(60) + + # Check if the app server started correctly + if cls.app_server_process.poll() is not None: + stdout, stderr = cls.app_server_process.communicate() + print(f"App Server stdout: {stdout}") + print(f"App Server stderr: {stderr}") + raise RuntimeError('App server failed to start') + + @classmethod + def tearDownClass(cls): + # Terminate the WSGI server process + cls.wsgi_server_process.terminate() + cls.wsgi_server_process.wait(timeout=60) + wsgi_exit_code = cls.wsgi_server_process.poll() + print(f"WSGI Server Exit Code: {wsgi_exit_code}") + + # Terminate the app server process + cls.app_server_process.terminate() + cls.app_server_process.wait(timeout=60) + app_exit_code = cls.app_server_process.poll() + print(f"App Server Exit Code: {app_exit_code}") + + if wsgi_exit_code != 0: + wsgi_pid = cls.wsgi_server_process.pid + # Forcefully kill all processes listening on port 8002 + subprocess.run(['pkill', '-f', f':{wsgi_pid}']) + assert wsgi_exit_code == 0, f"WSGI Server termination failed with exit code {wsgi_exit_code}" + else: + assert wsgi_exit_code == 0, f"WSGI Server termination failed with exit code {wsgi_exit_code}" + if app_exit_code != 0: + app_pid = cls.app_server_process.pid + # Forcefully kill all processes listening on port 5002 + subprocess.run(['pkill', '-f', f':{app_pid}']) + assert app_exit_code == 0, f"App Server termination failed with exit code {app_exit_code}" + else: + assert app_exit_code == 0, f"App Server termination failed with exit code {app_exit_code}" + + def check_server(self, endpoint, port): + # http://127.0.0.1:8000/hello/?content-type=application%2Fjson + response = requests.get(f'http://127.0.0.1:{port}/{endpoint}/?content-type=application%2Fjson') + assert response.status_code == 200 + assert "status" in response.json().keys() + + def test_wsgi_internal_server(self): + # Check the WSGI server + self.check_server('hello', 8002) + + def test_app_server(self): + # Check the app server + self.check_server('hello', 5002) \ No newline at end of file diff --git a/tests/test_limiters.py b/tests/test_limiters.py new file mode 100644 index 00000000..7f2c87e0 --- /dev/null +++ b/tests/test_limiters.py @@ -0,0 +1,55 @@ +"""Tests for the rate limiting code +Currently limited to just testing that it works and keeps to the requested time. +This code relies on the /hello/limit endpoint and relies on it's 1 access per- +second limit. +""" + +# Import necessary packages +import time +import pytest +from flask import g +from rest_VariantValidator.utils.limiter import limiter +from rest_VariantValidator.app import application # Import your Flask app + +# Fixture to set up the test client +@pytest.fixture(scope='module',name='client') +def rate_limit_test_client(): + """Return a test client that works for rate limiting""" + application.testing = False + application.debug = False + application.config['PROPAGATE_EXCEPTIONS'] = True + # This is fragile, and previous workarounds have failed before, so may + # break on updates to the limiter or flask test client code. + with application.app_context(): + setattr(g, '%s_rate_limiting_complete' % limiter._key_prefix, False) + test_client = application.test_client() # Create a test client to interact with the app + + yield test_client # This is where the tests will run + + +# Test the limiter works as intended, +def test_limit_endpoint_error_immediate(client): + """Provoke a limiter error by repeated requests to a limited endpoint""" + response = client.get('/hello/limit', headers={'Content-Type': 'application/json'}) + response = client.get('/hello/limit', headers={'Content-Type': 'application/json'}) + assert response.status_code == 429 + assert response.json["message"] == "Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org" + +def test_limit_endpoint_error_delayed(client): + """ + Provoke a limiter error by repeated requests to a limited endpoint, + but with a partial (sub 1 second) delay. + """ + response = client.get('/hello/limit', headers={'Content-Type': 'application/json'}) + time.sleep(0.5) + response = client.get('/hello/limit', headers={'Content-Type': 'application/json'}) + assert response.status_code == 429 + assert response.json["message"] == "Rate limit hit for this endpoint: See the endpoint documentation at https://rest.variantvalidator.org" + +def test_limit_endpoint_success(client): + """Repeat the same as above, but with a 1 second delay to exceed limit""" + response = client.get('/hello/limit', headers={'Content-Type': 'application/json'}) + time.sleep(1) + response = client.get('/hello/limit', headers={'Content-Type': 'application/json'}) + assert response.status_code == 200 + diff --git a/tests/test_wsgi_gateway.py b/tests/test_wsgi_gateway.py new file mode 100644 index 00000000..2efbb49e --- /dev/null +++ b/tests/test_wsgi_gateway.py @@ -0,0 +1,16 @@ +from rest_VariantValidator.wsgi import app +import pytest + + +@pytest.fixture(scope='module') +def client(): + app.testing = True + return app.test_client() # Create a test client to interact with the app + + +# Test function for the /hello/ endpoint +def test_wsgi_gateway(client): + response = client.get('/hello/') # Send a GET request to the /hello/ endpoint + assert response.status_code == 200 # Check if the response status code is 200 OK + assert response.json["status"] == "hello_world" # Check the JSON response content + assert "metadata" in response.json.keys() # Check if "metadata" key is in the JSON response diff --git a/vdb_docker.df b/vdb_docker.df deleted file mode 100644 index 68687d71..00000000 --- a/vdb_docker.df +++ /dev/null @@ -1,13 +0,0 @@ -FROM mysql:latest - -ENV MYSQL_RANDOM_ROOT_PASSWORD yes - -ENV MYSQL_DATABASE validator - -ENV MYSQL_USER vvadmin - -ENV MYSQL_PASSWORD var1ant - -RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* - -RUN wget https://leicester.figshare.com/ndownloader/files/16237784 -O /docker-entrypoint-initdb.d/validator_2019-07-10.sql.gz \ No newline at end of file diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 00000000..6731baea --- /dev/null +++ b/wsgi.py @@ -0,0 +1,19 @@ +""" +mod_wsgi gateway wsgi file +""" + +from rest_VariantValidator.app import application as application +from configparser import ConfigParser +from VariantValidator.settings import CONFIG_DIR + +config = ConfigParser() +config.read(CONFIG_DIR) +if config["logging"]["log"] == "True": + application.debug = True + application.config['PROPAGATE_EXCEPTIONS'] = True +else: + application.debug = False + application.config['PROPAGATE_EXCEPTIONS'] = False + +if __name__ == '__main__': + application.run(host="127.0.0.1", port=8000)