Skip to content

Commit d0503d3

Browse files
scaldingscalding
committed
Initial code commit
1 parent 4af87e7 commit d0503d3

File tree

71 files changed

+9013
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+9013
-1
lines changed

Diff for: ‎README.md

+15-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,16 @@
11
openregistry-loader
2-
===================
2+
=============================
3+
4+
Spring Batch based Open Registry loader
5+
6+
Jasig OpenRegistry
7+
https://wiki.jasig.org/display/OR/Home
8+
9+
Spring Batch
10+
http://static.springsource.org/spring-batch/
11+
12+
## Prerequisites
13+
14+
* JDK 1.7+
15+
16+
## The more detailed docs are in the `doc` directory.

Diff for: ‎bin/sor-data-loader.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/sh
2+
3+
if [ -z ${JAVA_HOME} ]
4+
then
5+
echo "Please set JAVA_HOME"
6+
exit 1
7+
fi
8+
9+
export LOADER_HOME=`cd $(dirname $0)/..;pwd`
10+
JAVA_CMD=${JAVA_HOME}/bin/java
11+
DEBUG_OPTS=" -agentlib:jdwp=transport=dt_socket,server=y,address=5000"
12+
13+
cd $LOADER_HOME
14+
15+
if [ $# -ne 1 ]
16+
then
17+
echo "usage: ./sor-data-loader.sh <sorId>"
18+
exit 1
19+
fi
20+
SOR_ID=$1
21+
22+
${JAVA_CMD} -cp "${LOADER_HOME}/etc:lib/*" -DLOADER_HOME=$LOADER_HOME org.springframework.batch.core.launch.support.CommandLineJobRunner jobs/sor-load-job.xml sorLoadJob -next sor=$SOR_ID

Diff for: ‎build.gradle

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
import groovy.xml.DOMBuilder
2+
3+
import java.nio.file.FileSystems
4+
import com.petebevin.markdown.MarkdownProcessor
5+
import org.xhtmlrenderer.pdf.ITextRenderer
6+
7+
defaultTasks 'clean', 'createDist'
8+
9+
apply plugin: 'idea'
10+
11+
allprojects {
12+
apply plugin: "groovy"
13+
apply plugin: "java"
14+
apply plugin: "maven"
15+
16+
group = "org.jasig.openregistry"
17+
version = "0.1-SNAPSHOT"
18+
19+
sourceCompatibility = '1.7'
20+
targetCompatibility = '1.7'
21+
22+
repositories {
23+
mavenCentral()
24+
maven { url "file://${FileSystems.default.getPath(System.getProperty("user.home"), ".m2/repository").toString()}" }
25+
flatDir {
26+
dirs 'lib'
27+
}
28+
}
29+
30+
ext {
31+
springVersion = "3.2.4.RELEASE"
32+
springBatchVersion = "2.2.1.RELEASE"
33+
openregistryVersion = "0.9.2.1"
34+
slf4jVersion = "1.6.1"
35+
postgresJdbcVersion = "9.1-901.jdbc4"
36+
oracleJdbcVersion = "12.1.0.1"
37+
groovyVersion = "2.1.5"
38+
}
39+
40+
sourceSets.main.java.srcDirs = []
41+
sourceSets.main.groovy.srcDirs = ['src/main/java', 'src/main/groovy']
42+
43+
44+
sourceSets.test.java.srcDirs = []
45+
sourceSets.test.groovy.srcDirs = ['src/test/java', 'src/test/groovy']
46+
}
47+
48+
project.ext {
49+
artifactId = "loader"
50+
}
51+
52+
task wrapper(type: Wrapper) {
53+
gradleVersion = '1.7'
54+
}
55+
56+
buildscript {
57+
repositories {
58+
mavenCentral()
59+
}
60+
dependencies {
61+
classpath "org.markdownj:markdownj:0.3.0-1.0.2b4"
62+
classpath "org.xhtmlrenderer:flying-saucer-pdf:9.0.2"
63+
}
64+
}
65+
66+
dependencies {
67+
//openregistry
68+
['openregistry-api', 'openregistry-service-impl', 'openregistry-sor-repository-xml', 'openregistry-repository-jpa-impl'].each {
69+
compile("org.jasig.openregistry:${it}:${project.openregistryVersion}") {
70+
exclude module: "spring-tx"
71+
}
72+
}
73+
74+
// spring batch
75+
compile("org.springframework.batch:spring-batch-core:${project.springBatchVersion}") {
76+
exclude module: "spring-tx"
77+
}
78+
79+
// spring
80+
['beans', 'context', 'tx'].each {
81+
compile "org.springframework:spring-${it}:${project.springVersion}"
82+
}
83+
84+
// BoneCP jdbc connection pool
85+
compile "com.jolbox:bonecp:0.7.1.RELEASE"
86+
87+
compile "log4j:log4j:1.2.17"
88+
89+
// slf4j stuff
90+
compile "org.slf4j:slf4j-api:${project.slf4jVersion}"
91+
compile "org.slf4j:jcl-over-slf4j:${project.slf4jVersion}"
92+
compile "org.slf4j:slf4j-log4j12:${project.slf4jVersion}"
93+
94+
//JDBC
95+
runtime "postgresql:postgresql:${project.postgresJdbcVersion}"
96+
runtime "oracle:ojdbc7:${project.oracleJdbcVersion}"
97+
98+
compile "org.codehaus.groovy:groovy-all:${project.groovyVersion}"
99+
compile("org.grails:grails-spring:2.2.4") {
100+
exclude module: "grails-bootstrap"
101+
exclude module: "gant_groovy1.8"
102+
exclude module: "javax.servlet-api"
103+
exclude module: "hibernate-jpa-2.0-api"
104+
exclude module: "slf4j-api"
105+
exclude module: "jcl-over-slf4j"
106+
exclude module: "spring-web"
107+
exclude module: "spring-tx"
108+
}
109+
}
110+
111+
task generatePDFDocument() {
112+
outputs.upToDateWhen { false }
113+
outputs.file new File(temporaryDir, "readme.pdf")
114+
doLast {
115+
def e = new File(temporaryDir, "readme.pdf")
116+
def processor = new MarkdownProcessor()
117+
def docString = ["doc/setup.md", "doc/configuration.md"].collect { processor.markdown(project.file(it).text) }.join('')
118+
def doc = DOMBuilder.parse(new StringReader("<html><title>Open Registry Loader</title><body>${docString}</body></html>"))
119+
def renderer = new ITextRenderer()
120+
renderer.setDocument(doc, null)
121+
renderer.layout()
122+
renderer.createPDF(e.newOutputStream())
123+
}
124+
}
125+
126+
task createDist(type: Zip, dependsOn: ':build') {
127+
def baseDir = { archiveName - ".zip" }
128+
into(baseDir) {
129+
into("etc") {
130+
from(project.file("etc"))
131+
from(project.file("src/main/resources")) {
132+
include "log4j.xml"
133+
}
134+
}
135+
into("lib") {
136+
from(jar)
137+
from(project.configurations.runtime)
138+
}
139+
into("bin") {
140+
from(project.file("bin")) {
141+
fileMode 0755
142+
}
143+
}
144+
into("doc") {
145+
from generatePDFDocument
146+
}
147+
}
148+
}

Diff for: ‎doc/configuration.md

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Configuration File Syntax
2+
The overall syntax consists of key-value pairs:
3+
4+
key "value"
5+
key2 "value2"
6+
7+
When a key needs a group of values, `{...}` is used:
8+
9+
key {
10+
key2 "value2"
11+
key3 "value3"
12+
}
13+
key4 "value4"
14+
15+
The grouping notation (`{...}`) can also denote a function (written in [groovy](http://groovy.codehaus.org/)) where
16+
appropriate:
17+
18+
key {
19+
converter { value -> value.toUpperCase() }
20+
}
21+
22+
# Database View System of Record Configuration
23+
Configuration of the loader is made from the perspective of Open Registry (OR) and the Loader rather than the source
24+
system, roughly following the object model in OR for `org.openregistry.core.domain.sor.SorPerson`. The configuration is
25+
structurally hierarchical. Each System of Record will have a `configuration` section. Each configuration section will
26+
have either an `id` field (in the instance of `configuration`) or a `keyField` that denotes each distinct property.
27+
Because of the hierarchy, the keys are cumulative (e.g. for a name, the `keyField`s for both the `names` element and
28+
`person` element identify a name.
29+
30+
## field
31+
Fields (leaves in configuration) can either be configured with a string denoting the column or a `fieldConfiguration`
32+
group. A `fieldConfiguration` can have two fields (from three possible):
33+
34+
* value: String: name of the column from which to get the value
35+
* staticValue: String: a static value
36+
* converter: function: if not using a staticValue, the final value is the value returned from the function. The value
37+
from the table cell is passed as an argument to the function. ex: `{ value -> value.toUpperCase() }`
38+
39+
Only one of `value` and `staticValue` can be used. `converter` is only used if `value` is used.
40+
41+
## Database View System of Record Configuration Items
42+
* configuration: group:
43+
* id (required): String: id for the SOR
44+
* person (required): group: person configuration
45+
* keyField (required): field: key field distinguishing each distinct person.
46+
* dateOfBirth: field: Date: date of birth
47+
* gender: field: String: gender. Valid values are M, F and `null`
48+
* ssn: field: String: social security number
49+
* names (required): group: names configuration
50+
* keyField (required): field: key field distinguishing each distinct name. Should be the type of the name.
51+
* given (required): field: given name
52+
* middle: field: middle name or initial
53+
* family: field: family name
54+
* prefix: field: name prefix
55+
* suffix: field: name suffix
56+
* roles: group: roles configuration
57+
* keyField (required): field: key field to distinguish each distinct role
58+
* affiliation (required): field: String: affiliation type
59+
* sorId (required): field: String: ID of the role in the SOR
60+
* start (required): field: Date: start date of the role
61+
* end: field: Date: end date of the role
62+
* title (required): field: String: role title
63+
* organizationalUnit (required): field: String: organizational unit for the role
64+
* sponsorId (required): field: Long: the id of the person responsible for this users role (supervisor)
65+
* sponsorType (required): field: String: type of sponsor
66+
* personStatus (required): field: String: status of the role (eg 'Active', 'Inactive')
67+
* percentage: field: Long:
68+
* terminationReson: field: String:
69+
* addresses: group:
70+
* keyField: field:
71+
* type: field: String:
72+
* bldgNo: field: String:
73+
* city: field: String:
74+
* country: field: String:
75+
* line1: field: String:
76+
* line2: field: String:
77+
* line3: field: String:
78+
* postalCode: field: String:
79+
* region: field: String:
80+
* roomNo: field: String:
81+
* updateDate: field: Date:
82+
* emailAddresses: group:
83+
* keyField: field:
84+
* type: field: String:
85+
* url: field: String:
86+
* leaves: group:
87+
* keyField: field:
88+
* end: field: Date:
89+
* reason: field: String:
90+
* start: field: Date:
91+
* phones: group:
92+
* keyField: field:
93+
* addressType: field: String:
94+
* areaCode: field: String:
95+
* countryCode: field: String:
96+
* extension: field: String:
97+
* number: field: String:
98+
* phoneLineOrder: field: Integer:
99+
* phoneType: field: String:
100+
* updateDate: field: Date:
101+
* urls: group:
102+
* keyField: field:
103+
* type: field: String:
104+
* url: field: String:
105+
* localAttribute: group: configuration for a localAttribute. There may be multiple of these in a person
106+
* keyField (required): field: key field distinguishing the attribute. Should be the key of the `localAttribute` set on
107+
the person.
108+
* value (required): field: value for the local attribute
109+
* disclosureSettings: group: disclosure settings configuration
110+
* disclosureCode: field: String:
111+
* withinGracePeriod: field: boolean:
112+
* lastUpdateDate: field: Date:
113+
114+
# Sample
115+
configuration {
116+
id "HR"
117+
person {
118+
keyField "empl_id"
119+
names {
120+
keyField {
121+
staticValue "Formal"
122+
}
123+
given "first_name"
124+
middle "middle_name"
125+
family "last_name"
126+
}
127+
localAttribute {
128+
keyField {
129+
staticValue "SSNLAST5"
130+
}
131+
value "ssn_last5_no"
132+
}
133+
roles {
134+
keyField "home_dept_cd"
135+
affiliation {
136+
staticValue "Staff"
137+
}
138+
start "appt_begin_dt"
139+
end "termination_dt"
140+
title {
141+
staticValue "king"
142+
}
143+
organizationalUnit "home_dept_cd"
144+
sponsorId {
145+
staticValue 1l
146+
}
147+
sponsorType {
148+
staticValue "Person"
149+
}
150+
personStatus {
151+
value "termination_dt"
152+
converter { value -> value ? 'Inactive' : 'Active' }
153+
}
154+
sorId {
155+
value "home_dept_cd"
156+
converter { value -> UUID.randomUUID().toString() }
157+
}
158+
}
159+
}
160+
}

Diff for: ‎doc/setup.md

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Open Registry Loader set up and deployment guide
2+
3+
## Checklist for setting up and running OR Loader SOR jobs
4+
5+
* Unzip the distribution archive into a filesystem directory of choice. Example: `/opt/openregistry-loader-0.1-SNAPSHOT`. This directory will be referred to as `$OR_LOADER_DIST_HOME`
6+
7+
* Create loader configuration directory `/etc/openregistry`. This directory will be referred to as `$OR_LOADER_CONFIG_HOME`
8+
9+
* Run the `$OR_LOADER_DIST_HOME/etc/dml/wipe_and_load_oracle.sql` DML script (IMPORTANT! - only done once, initially.
10+
Otherwise, ALL the DB objects with its data will be dropped and freshly re-created!) to create Spring Batch schema
11+
objects, Open Registry schema objects (with its local reference data), OR Loader schema objects.
12+
13+
* Copy `$OR_LOADER_DIST_HOME/etc/openregistry/config` and `$OR_LOADER_DIST_HOME/etc/openregistry/sor-repository` directories to `$OR_LOADER_CONFIG_HOME`
14+
15+
* Configure JDBC connection properties in `$OR_LOADER_CONFIG_HOME/config/openregistry.properties` with values of the local target Oracle RDBMS. Example config of local development instance of Oracle. For example:
16+
17+
`jdbc.driver.classname=oracle.jdbc.OracleDriver`
18+
19+
`jdbc.url=jdbc:oracle:thin:@localhost:1521:orcl`
20+
21+
`db.username=ORC`
22+
23+
`db.password=ORC`
24+
25+
* Configure mappings for each SOR in `$OR_LOADER_CONFIG_HOME/config/config.groovy`.
26+
27+
* Execute the batch job for each configured SOR by running the following script and passing an SOR name parameter. For example: `$OR_LOADER_DIST_HOME/bin/sor-data-loader.sh HR` or `$OR_LOADER_DIST_HOME/bin/sor-data-loader.sh SIS`
28+
29+
* Verify the correctness of the entire inbound data from HR processing by examining Open Registry `PRS_*` and `PRC_*` tables.
30+
31+
32+
33+
34+

0 commit comments

Comments
 (0)