From fa53bb5f343ca6193593d98a3f7a89b2ca8d81c1 Mon Sep 17 00:00:00 2001 From: Raoul Jean Pierre Bonnal Date: Thu, 25 Feb 2016 14:02:23 +0100 Subject: [PATCH 1/4] Introduce the place holder. Fix a bug with the processing of the when there are multiple instances # I suspect that if there are multiple SomethingAttached # only the second placeholder will be subsituted, the first one will not. # for I changed using the unless and try to substitute for the simpler placeholder # in any case. --- lib/bio/pipengine/job.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/bio/pipengine/job.rb b/lib/bio/pipengine/job.rb index 34a0d98..6777135 100644 --- a/lib/bio/pipengine/job.rb +++ b/lib/bio/pipengine/job.rb @@ -161,14 +161,28 @@ def sub_placeholders(cmd,sample,step=nil) tmp_cmd = cmd.gsub(//,sample.name) if tmp_cmd =~// sample_path_glob = (tmp_cmd.scan(/(\S+)/).map {|e| e.first}) - if sample_path_glob.empty? - tmp_cmd.gsub!(//,sample.path.join("\s")) - else + unless sample_path_glob.empty? sample_path_glob.each do |append| tmp_cmd.gsub!(/#{Regexp.quote(append)}/,(sample.path.map {|s| s+append}).join("\s")) end end + tmp_cmd.gsub!(//,sample.path.join("\s")) + end + + if tmp_cmd =~// + sample_group_glob = (tmp_cmd.scan(/(\S+)/).map {|e| e.first}) + unless sample_group_glob.empty? + sample_group_glob.each do |append| + tmp_cmd.gsub!(/#{Regexp.quote(append)}/,(sample.group+append)) + end + end + tmp_cmd.gsub!(//,sample.group) + end + + if tmp_cmd =~// + tmp_cmd = tmp_cmd.gsub(//,self.output+"/") end + # for resourcers and cpus tmp_cmd = sub_resources_and_cpu(tmp_cmd,step) From a1da991e14c4d4bfc69728828f1988ed425b4102 Mon Sep 17 00:00:00 2001 From: Raoul Jean Pierre Bonnal Date: Thu, 25 Feb 2016 17:39:54 +0100 Subject: [PATCH 2/4] qsub string was build in a wrong way, placing a "+" concat symbol that were not evaluated. --- VERSION | 2 +- bio-pipengine.gemspec | 2 +- lib/bio/pipengine/job.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index c18d72b..53a48a1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.1 \ No newline at end of file +0.8.2 \ No newline at end of file diff --git a/bio-pipengine.gemspec b/bio-pipengine.gemspec index f8fadc2..e243650 100644 --- a/bio-pipengine.gemspec +++ b/bio-pipengine.gemspec @@ -6,7 +6,7 @@ Gem::Specification.new do |s| s.name = "bio-pipengine" - s.version = "0.8.1" + s.version = "0.8.2" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.require_paths = ["lib"] diff --git a/lib/bio/pipengine/job.rb b/lib/bio/pipengine/job.rb index 6777135..61b3fcb 100644 --- a/lib/bio/pipengine/job.rb +++ b/lib/bio/pipengine/job.rb @@ -134,7 +134,7 @@ def to_script(options) end def submit - job_id = `qsub #{self.output}+"/"+#{self.name}.pbs` + job_id = `qsub #{self.output}"/"#{self.name}.pbs` @@logger.info "#{job_id}".green end From 4be789eea3025a8250fb8cde0a536980bc47cc69 Mon Sep 17 00:00:00 2001 From: Raoul Jean Pierre Bonnal Date: Fri, 5 Aug 2016 13:32:17 +0200 Subject: [PATCH 3/4] support stderr stdout group setting as PBS setting at job level --- README.md | 13 +++++++++++++ bin/pipengine | 1 + lib/bio-pipengine.rb | 2 ++ lib/bio/pipengine/job.rb | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/README.md b/README.md index 10a7e87..2aa5703 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ With this mode, PipEngine will submit pipeline jobs to the scheduler. directory names) --pbs-opts, -b : PBS options --pbs-queue, -q : PBS queue + --pbs-set-group, -w : PBS -W group_list (default: ) --inspect-pipeline, -i : Show steps --mail-exit, -a : Send an Email when the job terminates --mail-start, -r : Send an Email when the job starts @@ -680,6 +681,18 @@ within the job script. If a specific queue needs to be selected for sending the jobs to PBS, the ```--pbs-queue``` (short version **-q**) parameter can be used. This will pass to the ```qsub``` command the ```-q ``` taken from the command line. +Working in a team can be usefull to share the stdout and stderr of PBS, usually those files (also joined in one single file) have a restricted access just to the owner. To set the group of those files pipengine will automaticcally set the group to the usergroup (primary) by default, otherwise specifying a different group it will be setted accordibly. + +```shell +--pbs-set-group desired_group_name +``` + +will become, in the shell script: + +```shell +#PBS -W group_list=desired_group_name +``` + Copyright ========= diff --git a/bin/pipengine b/bin/pipengine index 36f0a1e..57411c8 100755 --- a/bin/pipengine +++ b/bin/pipengine @@ -31,6 +31,7 @@ when "run" opt :output_dir, "Output directory (override standard output directory names)", :short => "o", :type => :string opt :pbs_opts, "PBS options", :type => :strings, :short => "b" opt :pbs_queue, "PBS queue", :type => :string, :short => "q" + opt :pbs_set_group, "PBS -W group_list", :type => :string, :short => "w", :default => "" opt :inspect_pipeline, "Show steps", :short => "i", :type => :string opt :log, "Log script activities, by default stdin. Options are fluentd", :type => :string, :default => "stdin" opt :log_adapter, "(stdin|syslog|fluentd) In case of fluentd use http://destination.hostname:port/yourtag", :type => :string diff --git a/lib/bio-pipengine.rb b/lib/bio-pipengine.rb index d745481..888f343 100644 --- a/lib/bio-pipengine.rb +++ b/lib/bio-pipengine.rb @@ -13,3 +13,5 @@ require 'bio/pipengine/step' require 'bio/pipengine/job' require 'bio/pipengine' + +require 'etc' diff --git a/lib/bio/pipengine/job.rb b/lib/bio/pipengine/job.rb index 61b3fcb..a01980f 100644 --- a/lib/bio/pipengine/job.rb +++ b/lib/bio/pipengine/job.rb @@ -121,6 +121,11 @@ def to_script(options) file.puts "#PBS -N #{self.name}" file.puts "#PBS -d #{self.output}" file.puts "#PBS -q #{options[:pbs_queue]}" + if options[:pbs_set_group].empty? + file.puts "#PBS -W group_list=#{Etc.getgrgid(Etc.getpwnam(Etc.getlogin).gid).name}" + else + file.puts "#PBS -W group_list=#{options[:pbs_set_group]}" + end if options[:pbs_opts] file.puts "#PBS -l #{options[:pbs_opts].join(",")}" else From 53eaa95ce82bc1180141cc9545f4cb11cc9d9e3f Mon Sep 17 00:00:00 2001 From: Raoul Jean Pierre Bonnal Date: Fri, 5 Aug 2016 13:46:24 +0200 Subject: [PATCH 4/4] bump patch version, the newest pbs_set_group option is a minor improvement --- VERSION | 2 +- bio-pipengine.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 965065d..2bd77c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.3 +0.9.4 \ No newline at end of file diff --git a/bio-pipengine.gemspec b/bio-pipengine.gemspec index a91b253..63f8314 100644 --- a/bio-pipengine.gemspec +++ b/bio-pipengine.gemspec @@ -6,7 +6,7 @@ Gem::Specification.new do |s| s.name = "bio-pipengine" - s.version = "0.9.3" + s.version = "0.9.4" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.require_paths = ["lib"]