Skip to content

Commit

Permalink
AWS Gui + ImportMetadata Bug Fix (#127)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikesioda @msioda
mikesioda authored and msioda committed Jun 5, 2019
1 parent 9087dd5 commit 2d48f4d
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 97 deletions.
12 changes: 8 additions & 4 deletions script/biolockj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ docker_args() {
[ ${#gui} -gt 0 ] && args="g"
[ ${#config} -gt 0 ] && args="c=${config}" && [ ${#newPass} -gt 0 ] && args="${args} p=${newPass}"
[ ${#restart} -gt 0 ] && args="${args} r=${restart}"
[ ${#overrideBLJ} -gt 0 ] && args="${args} blj=${overrideBLJ}"
[ ${#overrideBLJ_SUP} -gt 0 ] && args="${args} blj_sup=${overrideBLJ_SUP}"
[ ${#blj} -gt 0 ] && args="${args} blj=${blj}"
[ ${#blj_sup} -gt 0 ] && args="${args} blj_sup=${blj_sup}"
echo "${args}"
}

Expand Down Expand Up @@ -75,13 +75,17 @@ parse_input_args() {
config=$(named_arg $args c)
newPass=$(named_arg $args p)
restart=$(named_arg $args r)
overrideBLJ=$(named_arg $args blj)
overrideBLJ_SUP=$(named_arg $args blj_sup)
blj=$(named_arg $args blj)
blj_sup=$(named_arg $args blj_sup)
[ ${#runDocker} -eq 0 ] && [ ${#blj} -gt 0 ] && echo "Arg \"blj\" is only applicable in Docker mode so will be ignored"
[ ${#runDocker} -eq 0 ] && [ ${#blj_sup} -gt 0 ] && echo "Arg \"blj_sup\" is only applicable in Docker mode so will be ignored"
[ ${#runDocker} -eq 0 ] && [ ${#runGui} -gt 0 ] && runDocker="d" && echo "Arg \"gui\" is only applicable in Docker mode --> starting Docker biolockj_controller container"
[ ${#runGui} -eq 0 ] && [ ${#restart} -eq 0 ] && [ ! -f "${config}" ] && ${BLJ}/script/biolockj -h && exit_script "Error [ biolockj ]: \"$config\" is not a valid file \n"
}

# Run biolockj by locally, on AWS, or in Docker mode based on script args
run_biolockj() {
[ ${#gui} -gt 0 ]
[ ${#runAws} -gt 0 ] && run_aws "${config}" && return
[ ${#runDocker} -gt 0 ] && dockblj $(docker_args) && return
[ ! -d "${BLJ_PROJ}" ] && exit_script "Error [ biolockj ]: Required env variable BLJ_PROJ undefined: \"${BLJ_PROJ}\""
Expand Down
6 changes: 3 additions & 3 deletions script/blj_test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
## ##
## Script requires valid BioLockJ Config file passed as a parameter. ##
## ##
## Pass optional param "-c" as 1st param to continue after any failure. ##
## Pass optional param "-x" as 1st param to continue after any failure. ##
## Otherwise exit script if any pipeline fails. ##
## ##
###########################################################################
Expand Down Expand Up @@ -53,5 +53,5 @@ verify_params() {
[ ! -d "$BLJ_PROJ" ] && exit_script "Error [ blj_test ]: Required env variable undefined --> \"BLJ_PROJ\""
}

verify_params
run_tests
verify_params $@
run_tests $@
166 changes: 82 additions & 84 deletions script/dockblj
View file
Original file line number Diff line number Diff line change
@@ -1,57 +1,53 @@
#!/bin/bash
##################################################################################
## Script launches BioLockJ via Docker + creates $DOCKER_CLONE_SCRIPT ##
## Bash env must include pipeline base directory: ${BLJ_PROJ} ##
## ##
## Required Parameters (If -gui arg passed, no other args required) ##
## Config file path: -c <file-path> ##
## ##
## Optional Parameters: ##
## Aws flag: -aws ##
## Use local BLJ flag: -blj ##
## Use local BLJ_SUP flag: -blj_sup ##
## Run GUI flag: -gui ##
## New email password: -p <new_password> ##
## Restart flag: -r <directory-path> ##
## ##
##################################################################################
###############################################################################
## Script launches BioLockJ via Docker + creates $DOCKER_CLONE_SCRIPT ##
## Bash env must include pipeline base directory: ${BLJ_PROJ} ##
## ##
## Required Parameters (If g arg passed, no other args required) ##
## Config file path: c <file_path> ##
## ##
## Optional Parameters: ##
## Aws flag: a ##
## Use local BLJ flag: -blj ##
## Use local BLJ_SUP flag: -blj_sup ##
## Run GUI flag: g ##
## New email password: p <new_password> ##
## Restart flag: r <dir_path> ##
## ##
###############################################################################
. "${DOCKER_LIB}"

DOCKER_ENV_FLAG="/.dockerenv"
GUI_PORT=8080

# Populate docker run env variable $BLJ_OPTIONS
# Always include vars to map Host paths: Config, $BLJ_PROJ, and $HOME.
# In AWS mode, add -aws indicator, otherwise include the Host path for pipeline input.
# In AWS mode, add a indicator, otherwise include the Host path for pipeline input.
# These paths are used by biolockj_controller to map Docker volumes for java_module containers
blj_options() {
options="-u ${HOME} -b $(blj_proj) -h $(hostname)"
$(var_not_null $overrideBLJ) && options="${options} -blj ${overrideBLJ}"
$(var_not_null $overrideBLJ_SUP) && options="${options} -blj_sup ${overrideBLJ_SUP}"
$(var_not_null $launchGui) && echo "${options}" && return

$(var_not_null $restartDir) && options="${options} -r ${BLJ_PROJ}/${restartDir#$(blj_proj)/}"
$(var_not_null $newPass) && options="${options} -p $newPass"
$(var_not_null $metaPath) && options="${options} -m $metaPath"
$(var_not_null $primerPath) && options="${options} -t $primerPath"

echo ${options} -i ${inputDir} -c ${config} ${runAws}
! $(null_var $blj) && options="${options} -blj ${blj}"
! $(null_var $blj_sup) && options="${options} -blj_sup ${blj_sup}"
! $(null_var $runGui) && echo "${options}" && return
! $(null_var $restartDir) && options="${options} r ${BLJ_PROJ}/${restartDir#$(blj_proj)/}"
! $(null_var $newPass) && options="${options} p $newPass"
! $(null_var $metaPath) && options="${options} -m $metaPath"
! $(null_var $primerPath) && options="${options} -t $primerPath"
echo "${options} -i ${inputDir} c ${config} ${runAws}"
}

# If dockblj called by biolockj_controller Docker container in GUI mode, set BLJ_PROJ based on the $BLJ_OPTIONS
# env var, otherwise return $BLJ_PROJ if called from a host machine. If dir ends with "/" trim it off
blj_proj() {
if [ -f "${DOCKER_ENV_FLAG}" ] && $(var_exists BLJ_OPTIONS); then
if [ -f "/.dockerenv" ] && $(var_exists BLJ_OPTIONS); then
bljProj=$(named_arg "${BLJ_OPTIONS}" -b)
else
bljProj="${BLJ_PROJ}"
fi

if [ "${bljProj: -1}" == "/" ]; then
len=$((${#bljProj}-1))
bljProj="${bljProj:0:len}"
fi
! $(var_not_null $bljProj) && exit_script "Error [ dockblj.blj_proj ]: Required env variable BLJ_PROJ undefined"
$(null_var $bljProj) && exit_script "Error [ dockblj.blj_proj ]: Required env variable BLJ_PROJ undefined"
echo "${bljProj}"
}

Expand All @@ -60,8 +56,7 @@ blj_proj() {
build_clone_script() {
img="$(dock_id)/biolockj_controller"
cmd=${@/$img/-ti $img /bin/bash}
$(var_not_null $launchGui) && cmd=${cmd/ npm start}

! $(null_var $runGui) && cmd=${cmd/npm start}
echo "#!/bin/bash" > "${DOCKER_CLONE_SCRIPT}"
echo "# This script launches a clone of the last biolockj_controller (same env vars + volumes)" >> "${DOCKER_CLONE_SCRIPT}"
echo "${cmd}" >> "${DOCKER_CLONE_SCRIPT}"
Expand All @@ -73,57 +68,72 @@ build_clone_script() {
# Get Docker image - add port, entrypoint, anc command if launching GUI
get_docker_img() {
cmd=$(dock_id)/biolockj_controller
$(var_not_null $launchGui) && echo "-p ${GUI_PORT}:3000 --expose ${GUI_PORT} -w /app/biolockj/web_app ${cmd} npm start"
! $(var_not_null $launchGui) && echo "${cmd}"
$(null_var $runGui) && echo "${cmd}"
! $(null_var $runGui) && echo "-p ${GUI_PORT}:3000 --expose ${GUI_PORT} -w /app/biolockj/web_app ${cmd} npm start"
}

# Get mapped Docker volumes
get_volumes() {
vols="-v ${DOCK_SOCK}:${DOCK_SOCK} -v ${HOME}:${BLJ_HOST_HOME}:delegated"
if $(var_not_null $runAws); then
vols="${vols} -v ${EFS}:${EFS}:delegated"
else
if $(null_var $runAws); then
vols="${vols} -v $(blj_proj):${EFS_PROJ}:delegated"
if $(var_not_null $launchGui); then
vols="${vols} -v ${BLJ}/resources/config/gui:${BLJ_CONFIG}:delegated"
else
if $(null_var $runGui); then
vols="${vols} -v $inputDir:${BLJ_INPUT}:ro -v $(dirname $config):${BLJ_CONFIG}:ro"
$(var_not_null $metaPath) && vols="${vols} -v $metaPath:${BLJ_META}:ro"
$(var_not_null $primerPath) && vols="${vols} -v $primerPath:${BLJ_PRIMER}:ro"
$(null_var $metaPath) && vols="${vols} -v $metaPath:${BLJ_META}:ro"
$(null_var $primerPath) && vols="${vols} -v $primerPath:${BLJ_PRIMER}:ro"
else
vols="${vols} -v ${BLJ}/resources/config/gui:${BLJ_CONFIG}:delegated"
fi

$(var_not_null $overrideBLJ) && vols="${vols} -v ${overrideBLJ}:/app/biolockj:ro"
$(var_not_null $overrideBLJ_SUP) && vols="${vols} -v ${overrideBLJ_SUP}:/app/blj_support:ro"
! $(null_var $blj) && vols="${vols} -v ${blj}:/app/biolockj:ro"
! $(null_var $blj_sup) && vols="${vols} -v ${blj_sup}:/app/blj_support:ro"
else
vols="${vols} -v ${EFS}:${EFS}:delegated"
fi
echo "${vols}"
}

# Boolean evaluation if single argument passed exists with a non-zero size
# Param 1 - Script arg
null_var() {
[ ${#1} -eq 0 ]
}

# Print and execute the docker run command with the correct volumes and env variables
run_docker() {
cmd="docker run --rm -e \"BLJ_OPTIONS=$(blj_options)\" $(get_volumes) $(get_docker_img)"
! $(null_var $runGui) && runInBak=" &"
cmd="docker run --rm -e \"BLJ_OPTIONS=$(blj_options)\" $(get_volumes) $(get_docker_img)${runInBak}"
printf "\n---------> Execute CMD [ ${cmd} ] \n\n"
build_clone_script "${cmd}"
docker run --rm -e "BLJ_OPTIONS=$(blj_options)" $(get_volumes) $(get_docker_img)
if $(null_var $runGui); then
docker run --rm -e "BLJ_OPTIONS=$(blj_options)" $(get_volumes) $(get_docker_img)
else
docker run --rm -e "BLJ_OPTIONS=$(blj_options)" $(get_volumes) $(get_docker_img) &
fi
}

# Read script args and bash env vars
# Param 1 - Array of dockblj script args
scan_script_and_env_args() {
launchGui=$(arg_exists "$@" -gui)
runAws=$(arg_exists "$@" -aws)
overrideBLJ=$(named_arg "$@" -blj)
overrideBLJ_SUP=$(named_arg "$@" -blj_sup)
! $(var_not_null $overrideBLJ) && overrideBLJ=$(arg_exists "$@" -blj) && [ ${#overrideBLJ} -eq 0 ] && overrideBLJ="${BLJ}"
! $(var_not_null $overrideBLJ_SUP) && overrideBLJ_SUP=$(arg_exists "$@" -blj_sup) && [ ${#overrideBLJ_SUP} -eq 0 ] && overrideBLJ_SUP="${BLJ_SUP}"
config=$(named_arg "$@" -c)
newPass=$(named_arg "$@" -p)
restartDir=$(named_arg "$@" -r)
inputDir=$(named_arg "$@" -i)
metaPath=$(named_arg "$@" -m)
primerPath=$(named_arg "$@" -t)
#inputDir="$(get_property $config input.dirPaths)"
#metaPath="$(get_property $config metadata.filePath)" && [ ${#metaPath} -gt 0 ] && metaPath="$(dirname $metaPath)"
#primerPath="$(get_property $config trimPrimers.filePath)" && [ ${#primerPath} -gt 0 ] && primerPath="$(dirname $primerPath)"
runGui=$(arg_exists "$@" g)
runAws=$(arg_exists "$@" a)
config=$(named_arg "$@" c)
newPass=$(named_arg "$@" p)
restartDir=$(named_arg "$@" r)
blj=$(named_arg "$@" -blj)
blj_sup=$(named_arg "$@" -blj_sup)

if $(null_var $blj) && blj=$(arg_exists "$@" -blj); then
blj="${BLJ}" && $(null_var $blj) && echo "Arg \"blj\" will be ignored because environment variable \"BLJ\" is undefined"
fi
if $(null_var $blj_sup) && blj_sup=$(arg_exists "$@" -blj_sup); then
blj_sup="${BLJ_SUP}" && $(null_var $blj_sup) && echo "Arg \"blj_sup\" will be ignored because environment variable \"BLJ_SUP\" is undefined"
fi

if $(null_var $runGui); then
inputDir="$(get_property $config input.dirPaths)"
metaPath="$(get_property $config metadata.filePath)" && [ ${#metaPath} -gt 0 ] && metaPath="$(dirname $metaPath)"
primerPath="$(get_property $config trimPrimers.filePath)" && [ ${#primerPath} -gt 0 ] && primerPath="$(dirname $primerPath)"
fi
}

# Start the local browswer
Expand All @@ -141,12 +151,6 @@ startBrowser() {
fi
}

# Boolean evaluation if single argument passed exists with a non-zero size
# Param 1 - Script arg
var_not_null() {
[ ${#1} -gt 0 ]
}

# Verify host directory exists, if dockblj is not deployed inside a Docker container
# Param 1 - System path
verify_dir() {
Expand All @@ -161,24 +165,18 @@ verify_file() {

# Verify paths are valid, if dockblj is not run inside a Docker container
verify_inputs() {
$(var_not_null $inputDir) && dirPath=( ${inputDir//, } )
! $(var_not_null $inputDir) || [ ${#dirPath[@]} -gt 1 ] && exit_script "Error [ dockblj ]: The -i parameter must reference a single input directory"
! $(null_var $inputDir) && dirPath=( ${inputDir//, } )
$(null_var $inputDir) || [ ${#dirPath[@]} -gt 1 ] && exit_script "Error [ dockblj ]: The -i parameter must reference a single input directory"
verify_dir $dirPath
verify_dir $BLJ_PROJ
$(var_not_null $restartDir) && verify_dir $restartDir
$(var_not_null $config) && verify_file $config
$(var_not_null $metaPath) && verify_dir $metaPath
$(var_not_null $primerPath) && verify_file $primerPath
! $(null_var $restartDir) && verify_dir $restartDir
! $(null_var $config) && verify_file $config
! $(null_var $metaPath) && verify_dir $metaPath
! $(null_var $primerPath) && verify_file $primerPath
}

printf "\n ---------> Execute CMD [ dockblj $(echo ${@}) ]\n\n"
scan_script_and_env_args $@

if $(var_not_null $launchGui); then
# Execute run_docker in background to startBrowser
run_docker &
startBrowser
else
verify_inputs
run_docker
fi
$(null_var $runGui) && verify_inputs
run_docker
! $(null_var $runGui) && startBrowser
21 changes: 15 additions & 6 deletions src/biolockj/module/implicit/ImportMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,21 +135,30 @@ protected void buildNewMetadataFile() throws MetadataException {
* @return Formatted Sample ID column name
*/
protected String formatMetaId( final String sampleIdColumnName ) {
String colName = sampleIdColumnName;
final char c = colName.trim().toCharArray()[ 0 ];
String id = sampleIdColumnName;
final char c = id.trim().toCharArray()[ 0 ];
if( c == 65279 ) {
Log.warn( getClass(),
"Removed ZERO WIDTH NO-BREAK invisible character [ASCII 65279] from 1st cell in metadata file. " +
"For more details, see http://www.fileformat.info/info/unicode/char/feff/index.htm" );

final char[] chars = colName.trim().toCharArray();
final char[] chars = id.trim().toCharArray();
for( int i = 0; i < chars.length; i++ )
Log.debug( getClass(), "ID[" + i + "] = " + chars[ i ] );

colName = colName.substring( 1 );
Log.info( getClass(), "Updated ID = " + colName );
id = id.substring( 1 );
Log.info( getClass(), "Updated ID = " + id );
}
return colName;

if( id.endsWith( Constants.FASTA ) || id.endsWith( Constants.FASTQ ) ) {
id = id.substring( 0, id.length() - 6 );
Log.info( getClass(), "Updated ID = " + id );
}
if( id.endsWith( Constants.GZIP_EXT ) ) {
id = id.substring( 0, id.length() - 3 );
Log.info( getClass(), "Updated ID = " + id );
}
return id;
}

/**
Expand Down

0 comments on commit 2d48f4d

Please sign in to comment.