Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Data dispatcher implementation #13

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
7bcd145
created data dispatcher class, test page, and config file
LTrestka May 30, 2023
b30344f
changed data_dispatcher name
LTrestka May 31, 2023
f1a91b7
Data dispatcher test page now has the ability to log in with username…
LTrestka Jun 9, 2023
40793fc
Merge branch 'data_dispatcher_implementation' of https://github.com/f…
LTrestka Jun 9, 2023
991943b
merge conflict resolved
LTrestka Jun 9, 2023
939f438
* Data dispatcher implemented up to the point of submission.
LTrestka Jul 18, 2023
94d4551
Cleaned up UI, added stability and compatibility between sam and data…
LTrestka Jul 31, 2023
6ff236b
submission broker looks at all existing launches at once and quickly …
Aug 15, 2023
6fbc33d
Updated submission agent to fetch data dispatcher jobs
LTrestka Aug 15, 2023
f0757b4
Updates to submission agent to fix broken jobsub_id format.
LTrestka Aug 21, 2023
373e207
Added a "dataset only" option for dd submissions, which is automatica…
LTrestka Aug 22, 2023
04d10ce
Finishing touches with faster queries, updates to split types (to run…
LTrestka Aug 29, 2023
72d766c
Several minor Bug fixes
LTrestka Sep 2, 2023
56b37b8
LTrestka Oct 4, 2023
941afbb
added docs
LTrestka Oct 26, 2023
c82e1d7
Refined docs for github
LTrestka Nov 7, 2023
2c01090
added html
LTrestka Nov 7, 2023
f1e7486
Updates to dd login
LTrestka Nov 7, 2023
bd9171c
Added the ability to use campaign keywords in campaign/stage fields -…
LTrestka Nov 7, 2023
4c30b7e
* Added test split functionality
LTrestka Dec 8, 2023
05f0c0e
added files to gitignore
LTrestka Dec 8, 2023
ade1723
LTrestka Dec 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Updates to dd login
  • Loading branch information
LTrestka committed Nov 7, 2023
commit f1e74861f85e992ab255b5d42a5b0eb6b2c40587
14 changes: 9 additions & 5 deletions submission_broker/submission_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -782,7 +782,7 @@ def main():
"""

ap = argparse.ArgumentParser()
ap.add_argument("-c", "--config", default="./submission_agent.cfg")
ap.add_argument("-c", "--config", default=os.environ.get("WEB_CONFIG", "/home/poms/poms/submission_broker/submission_agent.cfg"))
ap.add_argument("-d", "--debug", action="store_true")
ap.add_argument("--since", type=str)
ap.add_argument("-t", "--test", action="store_true", default=False)
Expand All @@ -794,17 +794,21 @@ def main():
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(filename)s:%(lineno)s:%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(filename)s:%(lineno)s:%(message)s")


config = args.config
if not config:
config = os.environ.get("WEB_CONFIG", "/home/poms/poms/submission_broker/submission_agent.cfg")
LOGIT.info("Submission Agent | Begin | Config: %s" % config)
if args.test:
agent = Agent(poms_uri="http://127.0.0.1:8080", submission_uri=os.environ["SUBMISSION_INFO"], config=args.config)
agent = Agent(poms_uri="http://127.0.0.1:8080", submission_uri=os.environ["SUBMISSION_INFO"], config=config)
for exp in agent.elist:
agent.check_submissions(exp, since=args.since)
elif args.one_time:
agent = Agent(config=args.config)
agent = Agent(config=config)
for exp in agent.elist:
agent.check_submissions(exp, since=args.since)
else:
agent = Agent(config=args.config)
agent = Agent(config=config)
agent.poll(since=args.since)


Expand Down
2 changes: 1 addition & 1 deletion webservice/Ctx.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def __init__(
self.role = role if role else pathv[3] if len(pathv) >= 4 else cherrypy.request.params.get("role", None)

self.username = username if username else get_user()
rows = self.db.execute("select txid_current();")
rows = self.db.execute(text("select txid_current();"))
for r in rows:
self.dbtransaction = r[0]
rows.close()
Expand Down
1 change: 0 additions & 1 deletion webservice/DMRService.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
config.read(os.environ.get("WEB_CONFIG","/home/poms/poms/webservice/poms.ini"))
PIPE = -1


def timestamp_to_readable(timestamp):
if type(timestamp) != type(datetime.datetime):
readable_format = timestamp.strftime("%A, %B %d, %Y")
Expand Down
6 changes: 3 additions & 3 deletions webservice/StagesPOMS.py
Original file line number Diff line number Diff line change
Expand Up @@ -629,8 +629,8 @@ def show_campaign_stages(self, ctx, campaign_ids=None, campaign_name=None, **kwa

csq = (
ctx.db.query(CampaignStage)
.options(joinedload("experiment_obj"))
.options(joinedload("campaign_obj"))
.options(joinedload(CampaignStage.experiment_obj))
.options(joinedload(CampaignStage.campaign_obj))
.options(joinedload(CampaignStage.experimenter_holder_obj))
.options(joinedload(CampaignStage.experimenter_creator_obj))
.options(joinedload(CampaignStage.experimenter_updater_obj))
Expand Down Expand Up @@ -1000,7 +1000,7 @@ def campaign_stage_submissions(self, ctx, campaign_name="", stage_name="", campa

tuples = (
ctx.db.query(Submission, SubmissionHistory, SubmissionStatus)
.join("experimenter_creator_obj")
.join(Submission.experimenter_creator_obj)
.filter(
Submission.campaign_stage_id.in_(campaign_stage_ids),
SubmissionHistory.submission_id == Submission.submission_id,
Expand Down
10 changes: 4 additions & 6 deletions webservice/poms_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,17 +267,15 @@ def activate_project(self, ctx, **kwargs):
return {"project_handles": retval, "msg": "Fail"}


@poms_method()
@poms_method(rtype="json")
def login_data_dispatcher(self, ctx, **kwargs):
# Assuming user is either logging in the first time, or signing in as someone else,
# hence, we will reinitialize the service and clear out any session info.
if kwargs.get('method', None) == 'password':
return ctx.dmr_service.login_with_password(kwargs.get('username', None), kwargs.get('password', None))
elif kwargs.get('method', None) == 'x509':
try:
return ctx.dmr_service.login_with_x509()
else:
except:
session_details = ctx.dmr_service.session_status()[1]
session_details['login_method'] = "Attempted login method: %s" % kwargs.get('method', None)
session_details['login_method'] = "Failed x509 login"
return json.dumps(session_details)

####################
Expand Down
10 changes: 5 additions & 5 deletions webservice/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import cherrypy
from cherrypy.process import plugins

from sqlalchemy import create_engine
from sqlalchemy import create_engine, text
from sqlalchemy.orm import scoped_session, sessionmaker
import sqlalchemy.exc

Expand Down Expand Up @@ -113,8 +113,8 @@ def bind_session(self):
cherrypy.request.dmr_service = self.dmr_service
try:
# Disabiling pylint false positives
self.session.execute("SET SESSION lock_timeout = '300s';") # pylint: disable=E1101
self.session.execute("SET SESSION statement_timeout = '400s';") # pylint: disable=E1101
self.session.execute(text("SET SESSION lock_timeout = '300s';")) # pylint: disable=E1101
self.session.execute(text("SET SESSION statement_timeout = '400s';")) # pylint: disable=E1101
self.session.commit() # pylint: disable=E1101
except sqlalchemy.exc.UnboundExecutionError:
# restart database connection
Expand All @@ -123,8 +123,8 @@ def bind_session(self):
cherrypy.engine.publish("bind", self.session)
cherrypy.request.db = self.session
self.session = scoped_session(sessionmaker(autoflush=True, autocommit=False, query_cls=FilterOutArchived))
self.session.execute("SET SESSION lock_timeout = '300s';") # pylint: disable=E1101
self.session.execute("SET SESSION statement_timeout = '400s';") # pylint: disable=E1101
self.session.execute(text("SET SESSION lock_timeout = '300s';")) # pylint: disable=E1101
self.session.execute(text("SET SESSION statement_timeout = '400s';")) # pylint: disable=E1101
self.session.commit() # pylint: disable=E1101

def release_session(self):
Expand Down
78 changes: 39 additions & 39 deletions webservice/static/js/data_dispatcher.js
Original file line number Diff line number Diff line change
Expand Up @@ -526,49 +526,49 @@ $(document).ready(function() {
}
});

$(`.login_submit`).on(`click`, (e) => {
$(`.login_submit`).on(`click`, async function(e){
e.preventDefault();
$.post(`/poms/login_data_dispatcher`,{
method: (e.target.id == 'login_password'?'password':'x509'),
username: $(`#username`).val(),
experiment: $('#session_experiment_id option:selected').html(),
password: $(`#password`).val(),
}, (data)=>{
data = JSON.parse(data);
if (data != null){
if (data.login_status != null){
$(`#login_status`).html(data.login_status);
if (data.login_status != `Logged in`){
await $.ajax({
url: `${pomspath}/login_data_dispatcher`,
type: 'GET',
dataType: 'json',
success: function(data){
data = JSON.parse(data);
if (data != null){
if (data.login_status != null){
$(`#login_status`).html(data.login_status);
if (data.login_status != `Logged in`){
cancelTimestampUpdates=true;
running=false;
$(`#login_method`).html(`N/A`);
$(`#signed-in-user`).html(`N/A`);
$(`#experiment`).html(`N/A`);
$(`#tokenLifespan`).val(``);
$(`#timeTillExpire`).html(`N/A`);
}
}
if(data.login_method != null){
$(`#login_method`).html(data.login_method);
}
if (data.dd_username != null){
$(`#signed-in-user`).html(data.dd_username);
}
if (data.dd_experiment != null){
$(`#experiment`).html(data.dd_experiment);
}
if (data.timestamp != null){
$(`#timeTillExpire`).html(`Loading.`);
showLoading();
cancelTimestampUpdates=true;
running=false;
$(`#login_method`).html(`N/A`);
$(`#signed-in-user`).html(`N/A`);
$(`#experiment`).html(`N/A`);
$(`#tokenLifespan`).val(``);
$(`#timeTillExpire`).html(`N/A`);
setTimeout(function() {
$(`#tokenLifespan`).val(data.timestamp);
cancelTimestampUpdates=false;
updateTimestamp();
}, 2000);
}
}
if(data.login_method != null){
$(`#login_method`).html(data.login_method);
}
if (data.dd_username != null){
$(`#signed-in-user`).html(data.dd_username);
}
if (data.dd_experiment != null){
$(`#experiment`).html(data.dd_experiment);
}
if (data.timestamp != null){
$(`#timeTillExpire`).html(`Loading.`);
showLoading();
cancelTimestampUpdates=true;
setTimeout(function() {
$(`#tokenLifespan`).val(data.timestamp);
cancelTimestampUpdates=false;
updateTimestamp();
}, 2000);
}
}
});
},
});
});


Expand Down
20 changes: 2 additions & 18 deletions webservice/templates/data_dispatcher_overview.html
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,11 @@ <h2 class="sub header">Method Called: {{method}}</h2>
{%block content %}
<input id="tokenLifespan" value="{{timestamp}}" hidden>
<div class="ui horizontal segments">
<div class="ui segment raised" style="width: 20%;">
<h3>Sign in with password</h3>
<div class=menu>
<div class="item">
<label for="username">Username</label>
<input id="username" type="text">
</div>
<div class="item">
<label for="password">Password</label>
<input id="password" type="password" >
</div>
<div class="item">
<button class="login_submit" id="login_password">Login</button>
</div>
</div>
</div>
<div class="ui segment raised" style="width: 20%;">
<h3>Sign in as poms</h3>
<h3>POMS Sign-in</h3>
<div class=menu>
<div class="item">
<button class="login_submit" id="login_token">Login</button>
<button class="login_submit" id="login_token">Reset Token</button>
</div>
</div>
</div>
Expand Down
2 changes: 1 addition & 1 deletion webservice/templates/login_setup_edit.html
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
<tfoot>
<tr>
<th colspan="5">
<button class="ui labled teal icon button" onclick="add_template();return false;") {{ 'disabled' if (session_role=='superuser') }}>
<button class="ui labled teal icon button" onclick="add_template();return false;" {{ 'disabled' if (session_role=='superuser') }}>
<i class="icon add square"></i>
Add
</button>
Expand Down