Skip to content
Snippets Groups Projects
Commit d2563be8 authored by Bienchen's avatar Bienchen
Browse files

Merge branch 'validation-tool' into develop

parents 6057eb2d cf1e8761
No related branches found
No related tags found
No related merge requests found
\.DS_Store
...@@ -2,5 +2,22 @@ ...@@ -2,5 +2,22 @@
This is a collection of tools, code and examples used for converting user's modelling projects into [ModelCIF format](https://mmcif.wwpdb.org/dictionaries/mmcif_ma.dic/Index/) for upload to [ModelArchive](https://www.modelarchive.org) (MA). The whole repository has grown over time from bulk uploads to MA, we assisted users with. Do not expect everything to work out of the box, old conversion projects may be outdated by now. This is a collection of tools, code and examples used for converting user's modelling projects into [ModelCIF format](https://mmcif.wwpdb.org/dictionaries/mmcif_ma.dic/Index/) for upload to [ModelArchive](https://www.modelarchive.org) (MA). The whole repository has grown over time from bulk uploads to MA, we assisted users with. Do not expect everything to work out of the box, old conversion projects may be outdated by now.
<!-- LocalWords: modelcif ModelArchive
## ModelCIF resources
The official documentation of [ModelCIF](https://mmcif.wwpdb.org/dictionaries/mmcif_ma.dic/Index/) and [PDBx/mmCIF](https://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Index/) can be found here: https://mmcif.wwpdb.org
ModelCIF dictionary files can be found in the official repository: https://github.com/ihmwg/ModelCIF
## Directories
A short overview of the directories in this repository and what they do.
|Path |Content |
|-----------|----------------------------------------------------------------|
|[projects/](projects/) |Collection of model conversions done for various user projects. |
|[validation/](validation/)|A tool to check the formatting of ModelCIF files. |
<!-- LocalWords: modelcif ModelArchive PDBx
--> -->
...@@ -3,3 +3,9 @@ line-length=80 ...@@ -3,3 +3,9 @@ line-length=80
[tool.pylint.REPORTS] [tool.pylint.REPORTS]
reports='no' reports='no'
[tool.pylint.MASTER]
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-allow-list='rapidjson'
\ No newline at end of file
ARG VERSION_PYTHON="3.7"
ARG VERSION_BASE_IMAGE="python:${VERSION_PYTHON}-alpine3.16"
FROM ${VERSION_BASE_IMAGE}
# We need to declare ARGs again which were declared before the build stage
# (FROM directive), otherwise they won't be available in this stage.
ARG VERSION_PYTHON
ARG VERSION_BASE_IMAGE
ARG VERSION_CPP_DICT_PACK="v2.500"
ARG VERSION_PY_MMCIF="0.76"
## Set up environment
ENV MMCIF_DICTS_DIR="/usr/local/share/mmcif-dict-suite" \
SRC_DIR="/tmp" \
VERSION_CPP_DICT_PACK=${VERSION_CPP_DICT_PACK} \
VERSION_BASE_IMAGE=${VERSION_BASE_IMAGE} \
VERSION_PYTHON=${VERSION_PYTHON} \
VERSION_PY_MMCIF=${VERSION_PY_MMCIF} \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1
LABEL org.modelarchive.base-image="${VERSION_BASE_IMAGE}"
LABEL org.modelarchive.cpp-dict-pack.version="${VERSION_CPP_DICT_PACK}"
LABEL maintainer="Stefan Bienert <stefan.bienert@unibas.ch>"
LABEL vendor1="Schwede Group (schwedelab.org)"
LABEL vendor2="SIB - Swiss Institute of Bioinformatics (sib.swiss)"
LABEL vendor3="Biozentrum - University of Basel (biozentrum.unibas.ch)"
## Install the RCSB CPP Dict Suite (only the binaries we need)
WORKDIR ${SRC_DIR}
RUN set -e pipefail; \
export DICT_PACK_SRC_DIR="${SRC_DIR}/cpp-dict-pack.git"; \
apk update; \
apk upgrade; \
apk add abuild binutils bison build-base cmake flex git gcc \
extra-cmake-modules tcsh; \
#
## Install the RCSB mmCIF Dict Suite
git clone -b ${VERSION_CPP_DICT_PACK} \
--single-branch --recurse-submodules \
https://github.com/rcsb/cpp-dict-pack.git \
${DICT_PACK_SRC_DIR}; \
mkdir ${DICT_PACK_SRC_DIR}/build; \
cd ${DICT_PACK_SRC_DIR}; \
cd ${DICT_PACK_SRC_DIR}/build; \
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON; \
make; \
for cif_tool in CifCheck DictToSdb; do \
mv bin/${cif_tool} /usr/local/bin; \
done; \
cd ${SRC_DIR}; \
rm -r ${DICT_PACK_SRC_DIR}; \
#
## Install the RCSB py-mmcif Python module
/usr/local/bin/python -m pip install --upgrade pip; \
/usr/local/bin/python -m pip install mmcif==${VERSION_PY_MMCIF} \
python-rapidjson; \
#
## Clean up/ remove unnecessary stuff
apk del abuild binutils bison build-base cmake flex git gcc \
extra-cmake-modules tcsh; \
apk add libstdc++
## Copy tools/ entrypoint script
COPY --chmod=755 get-mmcif-dict-versions.py \
/usr/local/bin/get-mmcif-dict-versions
COPY --chmod=755 entrypoint.sh /
COPY --chmod=755 validate-mmcif-file.py /usr/local/bin/validate-mmcif-file
## Create dictionaries for validating mmCIF files.
## The version to be created is controlled by build argument USE_DICT_VERSION.
## Use a versions vailable at
## https://github.com/ihmwg/ModelCIF/tree/master/archive or the special word
## "latest", that will use
## https://github.com/ihmwg/ModelCIF/blob/master/dist/mmcif_ma.dic.
## Dictionaries do not change that frequently therefore we skip the hassle of
## keeping them in an external volume.
ARG USE_DICT_VERSION="1.4.2"
ENV USE_DICT_VERSION=${USE_DICT_VERSION}
LABEL org.modelarchive.dict_release="${USE_DICT_VERSION}"
WORKDIR ${SRC_DIR}
RUN set -e pipefail; \
apk add curl; \
export _DICT_DIR="${SRC_DIR}/mmcif_dicts"; \
export _DICT_URL="https://mmcif.wwpdb.org/dictionaries/ascii"; \
export _GIT_URL="https://raw.github.com/ihmwg/ModelCIF/master"; \
export _MA_DICT_URL="${_GIT_URL}/archive/mmcif_ma-v${USE_DICT_VERSION}.dic"; \
mkdir ${_DICT_DIR}; \
mkdir ${MMCIF_DICTS_DIR}; \
cd ${_DICT_DIR}; \
#
## Fetch the dictionary definition language
curl ${_DICT_URL}/mmcif_ddl.dic.gz -s -o mmcif_ddl.dic.gz; \
gunzip *.gz; \
#
## Fetch the merged ModelCIF dictionary
if test x${USE_DICT_VERSION} == xlatest; then \
export _MA_DICT_URL="${_GIT_URL}/dist/mmcif_ma.dic"; \
fi; \
curl ${_MA_DICT_URL} -s -L -o mmcif_ma.dic; \
#
## Build the ModelCIF SDB
DictToSdb -ddlFile mmcif_ddl.dic \
-dictFile mmcif_ma.dic \
-dictSdbFile ${MMCIF_DICTS_DIR}/mmcif_ma.sdb; \
#
## Fetch the stable PDBx/mmCIF dictionary
curl ${_DICT_URL}/mmcif_pdbx_v50.dic -s -o mmcif_pdbx_v50.dic; \
#
## Build the PDBx/mmCIF SDB
DictToSdb -ddlFile mmcif_ddl.dic \
-dictFile mmcif_pdbx_v50.dic \
-dictSdbFile ${MMCIF_DICTS_DIR}/mmcif_pdbx_v50.dic.sdb; \
#
## Get versions of ModelCIF & PDBx/mmCIF dictionaries
get-mmcif-dict-versions --parent-location ${_GIT_URL}/base/mmcif_pdbx_v50.dic \
--child-location ${_MA_DICT_URL} \
mmcif_ma.dic; \
mv mmcif_ma_version.json ${MMCIF_DICTS_DIR}/; \
#
## Make SDBs readable and keep possible error logs from building them
mv *.log ${MMCIF_DICTS_DIR}/ 2>/dev/null || :; \
chmod o+r ${MMCIF_DICTS_DIR}/*; \
#
## Clean up
cd ${SRC_DIR}; \
rm -r ${_DICT_DIR}; \
apk del curl
# for development
#RUN set -e pipefail; \
# apk add bash emacs gcc build-base; \
# /usr/local/bin/python -m pip install pylint black; \
# apk del gcc build-base
## Add a dedicated user for mmCIF file validation
## MMCIF_USER_ID can be used to avoid file permission issues in development.
ARG MMCIF_USER_ID=501
RUN adduser -S -u ${MMCIF_USER_ID} mmcif-vldtr
USER mmcif-vldtr
ENTRYPOINT ["/entrypoint.sh"]
# have tool ready
# - entrypoint: validate... just runs validation, celery runs celery, CMD else
# write Python to run & check mmCIF
# - Note dictionary versions in the mmCIF file!
# for Celery:
# - depends_on without implementing the 'waits' in this entrypoint.sh:
# https://marcopeg.com/docker-compose-healthcheck/
# LocalWords: ENV DICTS SRC tmp schwedelab RCSB WORKDIR pipefail apk dev ARG
# LocalWords: ARGs
[[_TOC_]]
# ModelCIF validation tool
This is a tool to check that the formatting of ModelCIF files complies with the ModelCIF format declaration (aka "dictionary"). Upon successful validation, a ModelCIF file can be extended with the dictionary version the file was compared to (option [`--extend-validated-file`](#add-validation-dictionary-to-ModelCIF-file)). For more basic mmCIF validation, the dictionary of the underlying PDBx/mmCIF format is also available.
The easiest way to run validation is from Docker container.
The tool itself is a wrapper around the [`CifCheck`](https://github.com/rcsb/cpp-dict-pack) tool by [RCSB](https://www.rcsb.org/).
## How to run the validation tool
This is just a description of the validation tool itself. When running it from inside a container, the command needs to be prefixed with the instructions to start the container. Find information for running the validation container in "[How to run the container](#how-to-run-the-container)".
Upon completion, if there hasn't been any error running the command, the validation tool returns a concise report in JSON format. That output is meant to be input to a website or any kind of nicely formatted report. Output can also be stored as a file. If the tested ModelCIF file is fully compliant with the ModelCIF format, the JSON output has
- `status` "completed"
- no messages in the `diagnosis` list
- `versions` of the dictionaries the file was tested against
Format violations will be listed in `diagnosis`.
The most basic way to invoke the validation tool is just with a ModelCIF file (example shows the command plus possible output):
```bash
$ validate-mmcif-file model.cif
{"status":"completed","diagnosis":[],"versions":[{"title":"mmcif_pdbx_v50.dic","version":"5.358","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/base/mmcif_pdbx_v50.dic"},{"title":"mmcif_ma.dic","version":"1.4.0","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/dist/mmcif_ma.dic"}]}
$
```
### Add dictionary information used for validation to ModelCIF file
Since both dictionaries, ModelCIF and PDBx/mmCIF, represent actively developed file formats, different versions exist. While extending them, quite some thinking goes into making only non-breaking changes. The idea is that a ModelCIF file formatted following dictionary 1.3, is still valid with version 1.4. But the version number also tells you which feature to expect from a ModelCIF file, so it seems like a good idea to keep the version inside the file.
The validation tool can add the version upon positive validation, enabled by the `--extend-validated-file` (`-e`).
`-e` can take an alternative file name to write the validated ModelCIF file to, e.g. if one wants to keep the original ModelCIF file unaltered:
```bash
$ validate-mmcif-file -e validated_model.cif model.cif
{"status":"completed","diagnosis":[],"versions":[{"title":"mmcif_pdbx_v50.dic","version":"5.358","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/base/mmcif_pdbx_v50.dic"},{"title":"mmcif_ma.dic","version":"1.4.0","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/dist/mmcif_ma.dic"}]}
$
```
The last command will generate a new file `validated_model.cf` upon positive validation (`diagnosis` points to an empty list), with the `versions` added to the `_audit_conform` list inside the file.
To add the validation dictionaries to `_audit_conform` in the original ModelCIF file, just invoke `-e` without an alternative file name... well almost. By the way Python handles this kind of command line arguments, `-e` consumes everything after it, that does not start with a `-`, as a file name. So `validate-mmcif-file -e model.cif` would mean that `-e` assumes `model.cif` as its file name but then the command fails because it is missing the ModelCIF file to be validated. The solution is either putting `-e` at the beginning of the arguments list or after the ModelCIF file name at the very end, if there are no other command line arguments:
```bash
$ validate-mmcif-file model.cif -e
{"status":"completed","diagnosis":[],"versions":[{"title":"mmcif_pdbx_v50.dic","version":"5.358","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/base/mmcif_pdbx_v50.dic"},{"title":"mmcif_ma.dic","version":"1.4.0","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/dist/mmcif_ma.dic"}]}
$
```
### Base directory for associated files
For a ModelCIF file using the `_ma_entry_associated_files` category, the validation tool tries to merge associated data into the ModelCIF file, if `_ma_entry_associated_files.file_format` is `cif`. That way the outsourced data is validated, too.
Command line argument `--associates-dir` (`-a`) is used to declare the base directory associated files are stored in. Inside the directory, the path must follow what is defined in `_ma_entry_associated_files.file_url`. If the URL is just the file name, the file must be stored right in the associates directory. The following example works for `_ma_entry_associated_files.file_url model_pae.cif`
```bash
$ grep _ma_entry_associated_files.file_url model.cif
_ma_entry_associated_files.file_url model_pae.cif
$ ls extra
model_pae.cif
$ validate-mmcif-file -a extra model.cif
{"status":"completed","diagnosis":[],"versions":[{"title":"mmcif_pdbx_v50.dic","version":"5.358","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/base/mmcif_pdbx_v50.dic"},{"title":"mmcif_ma.dic","version":"1.4.0","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/dist/mmcif_ma.dic"}]}
$
```
If the URL points to a subdirectory, this must be reflected by the associates directory tree declared to the validation tool. The following example illustrates that the `extra` directory needs a `pae` directory storing the associated file as expected by `_ma_entry_associated_files.file_url`:
```bash
$ grep _ma_entry_associated_files.file_url model.cif
_ma_entry_associated_files.file_url pae/model_pae.cif
$ ls extra
pae
$ ls extra/pae
model_pae.cif
$ validate-mmcif-file -a extra model.cif
{"status":"completed","diagnosis":[],"versions":[{"title":"mmcif_pdbx_v50.dic","version":"5.358","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/base/mmcif_pdbx_v50.dic"},{"title":"mmcif_ma.dic","version":"1.4.0","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/dist/mmcif_ma.dic"}]}
$
```
### Misc. arguments
**`--help`** (**`-h`**) Print a help/ usage page for the validation tool.
**`--dict-sdb <SDB FILE>`** (**`-d`**) Format dictionary in (binary) SDB format used for validating a ModelCIF file. The container comes with a SDB for ModelCIF and one for the original PDBx/mmCIF format.
**`--out-file <JSON FILE>`** (**`-o`**) Instead of printing the output to `stdout`, store it in a JSON file.
**`--verbose`** (**`-v`**) Write information from intermediate steps to `stdout`. This includes the raw output of `CifCheck`.
## How to run the container
The call to the validation tool (almost) stays the same, it just needs instructions to start the Docker container as a prefix:
```bash
$ docker run --rm -v /home/user/models:/data registry.scicore.unibas.ch/schwede/mabakerimport/mmcif-dict-suite:dev validate-mmcif-file /data/model.cif
{"status":"completed","diagnosis":[],"versions":[{"title":"mmcif_pdbx_v50.dic","version":"5.358","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/base/mmcif_pdbx_v50.dic"},{"title":"mmcif_ma.dic","version":"1.4.0","location":"https://raw.githubusercontent.com/ihmwg/ModelCIF/a24fcfa8d6c3ceb4b6f0676bcc341ac0cd24ff1f/dist/mmcif_ma.dic"}]}
$
```
`docker run` is the call to execute a certain command inside a container. `--rm` makes sure that the container is removed from the system once the job completed.
Since the container has its own internal file system separated, `-v` is utilised to mount a directory from the host into the container.
- explain what command does
- explain volumes/ external mounts
- explain for -e
- explain for -a
## How to pull a copy of the container from our registry
- since we use it ourselves and are involved in the development, we usually notice when a new dictionary comes out
## How to build the container from scratch
# Files in this directory
<!-- LocalWords: PDBx ModelCIF TOC JSON CifCheck RCSB mmcif cif pdbx dic dir
LocalWords: url pae sdb SDB stdout
-->
#!/bin/sh
## (We use sh since Alpine does not have Bash by default)
## exit immediately on commands with a non-zero exit status.
set -euo pipefail
## When started without any arguments, "-h", "--help", "-help" or "help", print
## usage.
if [ $# -eq 0 ] || [ x$1 == x"-h" ] || [ x$1 == x"--help" ] ||
[ x$1 == x"-help" ] || [ x$1 == x"help" ]; then
echo " mmCIF file format validation tool."
echo "------------------------------------------"
echo "Provided by SWISS-MODEL / Schwede group"
echo "(swissmodel.expasy.org / schwedelab.org)"
echo ""
echo "This container checks that mmCIF files are"
echo "properly formatted according to the"
echo "MAX/ mmCIF dictionary. At the moment,"
echo "there is one tool available that acts as a"
echo "command line tool: validate-mmcif-file."
echo "For further usage information, call this"
echo "container executing"
echo "'validate-mmcif-file --help'."
exit 1
fi
exec "$@"
# LocalWords: euo pipefail eq Schwede schwedelab mmcif fi
#! /usr/local/bin/python
"""Get version and location of relevant mmCIF dictionaries for ModelCIF.
Fetch the versions of the ModelCIF dictionary and the PDBx/mmCIF dictionary used
to build it into a JSON file.
"""
# pylint: disable=invalid-name
# pylint: enable=invalid-name
import argparse
import sys
import rapidjson as json
from mmcif.io.IoAdapterPy import IoAdapterPy
def _parse_command_line():
"""Get arguments."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"dic_file",
type=str,
metavar="<DICTIONARY FILE>",
help="The mmCIF dictionary file to read the versions from.",
)
parser.add_argument(
"--parent",
"-p",
type=str,
metavar="<NAME OF PARENT DICT>",
help="Name of to the 'parent' dictionary. This is the one the other "
+ "dictionary is appended to. This is usually the mmcif_pdbx_v50.dic.",
default="mmcif_pdbx_v50.dic",
)
parser.add_argument(
"--output",
"-o",
type=str,
metavar="<PATH TO VERSION FILE>",
help="Path to store the JSON file with the version at.",
default="mmcif_ma_version.json",
)
parser.add_argument(
"--parent-location",
"-u",
type=str,
metavar="<URL OF PARENT DICT FILE>",
help="Download location of the parent dictionary file.",
default=None,
)
parser.add_argument(
"--child-location",
"-l",
type=str,
metavar="<URL OF CHILD DICT FILE>",
help="Download location of the child dictionary file.",
default=None,
)
opts = parser.parse_args()
return opts
def _error(msg):
"""Print a final error message."""
print(msg + "\nAborting.", file=sys.stderr)
sys.exit(1)
def _get_data_cat(cat, file_name, data):
"""Get a data category from a mmCIF data blob."""
obj = data.getObj(cat)
if obj is None:
_error(f"No '{cat}' object found in '{file_name}'.")
return obj
def _get_data_item(itm, cat, file_name, cat_data):
"""Get a single data item from a data category."""
val = cat_data.getAttributeValueList(itm)
if len(val) != 1:
_error(
f"Expected exactly 1 '{cat}.{itm}' in '{file_name}', "
+ f"found '{', '.join(val)}'."
)
return val[0]
def _get_versions(dic_file, parent_name, io_adapter):
"""Fetch the 'category_group_list' object and assemble a version for the
dictionary."""
dic = io_adapter.readFile(inputFilePath=dic_file)
# fetch a data container from the list returned by the adapter
cntnr = None
for obj in dic:
if "dictionary" in obj.getObjNameList():
cntnr = obj
break
if cntnr is None:
_error(f"No 'dictionary' object found in '{dic_file}'.")
dic = _get_data_cat("dictionary", dic_file, cntnr)
vrsn = _get_data_item("version", "dictionary", dic_file, dic)
ttl = _get_data_item("title", "dictionary", dic_file, dic)
dic_version = {"title": ttl, "version": vrsn}
cmp = _get_data_cat("pdbx_dictionary_component", dic_file, cntnr)
dc_idx = cmp.getAttributeIndex("dictionary_component_id")
vs_idx = cmp.getAttributeIndex("version")
for row in cmp:
if row[dc_idx] == parent_name:
vrsn = row[vs_idx]
prnt_version = {"title": parent_name, "version": vrsn}
break
return dic_version, prnt_version
def _add_dict_location(parent, child, parent_loc, child_loc):
"""Add URLs to the dictionary versions if available."""
if parent_loc is None:
parent["location"] = "."
else:
parent["location"] = parent_loc
if child_loc is None:
child["location"] = "."
else:
child["location"] = child_loc
def _main():
"""Run as script."""
opts = _parse_command_line()
io_adapter = IoAdapterPy(False, sys.stdout)
c_vrsn, p_vrsn = _get_versions(opts.dic_file, opts.parent, io_adapter)
_add_dict_location(
p_vrsn, c_vrsn, opts.parent_location, opts.child_location
)
with open(opts.output, "w", encoding="utf8") as jfh:
json.dump({"versions": [p_vrsn, c_vrsn]}, jfh)
if __name__ == "__main__":
_main()
# LocalWords: DictToSdb SDB PDBx CifCheck pylint mmcif pdbx dic nAborting
# LocalWords: macromolecular utf
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment