katolaz
/
scorsh
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

pipeline from spool to worker is done -- added examples

Browse Source
pull/1/head
KatolaZ 8 years ago
parent ed637037b7
commit 726b399e47
18 changed files with 537 additions and 125 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      commits.go
  2. 41
      config.go
  3. 30
      examples/create_spool_message.sh
  4. 29
      examples/scorsh_example.cfg
  5. 62
      examples/worker1/allowed_users.asc
  6. BIN
      examples/worker1/allowed_users.pgp
  7. 24
      examples/worker1/worker1.cfg
  8. 62
      examples/worker2/allowed_users.asc
  9. BIN
      examples/worker2/allowed_users.pgp
  10. 14
      examples/worker2/worker2.cfg
  11. 44
      sandpit/test_parse_message.go
  12. 26
      sandpit/test_regexp.go
  13. 63
      sandpit/test_worker_cfg.go
  14. 48
      scorsh.cfg
  15. 39
      scorsh.go
  16. 42
      spooler.go
  17. 86
      types.go
  18. 46
      workers.go

6
commits.go
Unescape View File

@ -53,9 +53,9 @@ func walk_commits(msg SCORSHmsg, w *SCORSHworker) error {
fmt.Printf("Inside parse_commits\n") fmt.Printf("Inside parse_commits\n")
reponame := msg.repo reponame := msg.Repo
old_rev := msg.old_rev old_rev := msg.Old_rev
new_rev := msg.new_rev new_rev := msg.New_rev
repo, err := git.OpenRepository(reponame) repo, err := git.OpenRepository(reponame)
if err != nil { if err != nil {

41
config.go
Unescape View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"bytes"
"fmt" "fmt"
"github.com/go-yaml/yaml" "github.com/go-yaml/yaml"
"io" "io"
@ -41,7 +40,7 @@ func ReadGlobalConfig(fname string) *SCORSHmaster {
} }
if cfg.LogPrefix != "" { if cfg.LogPrefix != "" {
log.SetPrefix(cfg.LogPrefix) log.SetPrefix(cfg.LogPrefix+ " ")
} }
// If the user has not set a spooldir, crash loudly // If the user has not set a spooldir, crash loudly
@ -54,46 +53,10 @@ func ReadGlobalConfig(fname string) *SCORSHmaster {
// Check if the user wants to redirect the logs to a file // Check if the user wants to redirect the logs to a file
// If we got so far, then there is some sort of config in cfg // If we got so far, then there is some sort of config in cfg
log.Printf("----- Starting SCORSH -----\n")
log.Printf("Successfully read config from %s\n", fname) log.Printf("Successfully read config from %s\n", fname)
return cfg return cfg
} }
func (cfg *SCORSHmaster) String() string {
var buff bytes.Buffer
buff.WriteString("spooldir: ")
buff.WriteString(cfg.Spooldir)
buff.WriteString("\nlogfile: ")
buff.WriteString(cfg.Logfile)
buff.WriteString("\nlogprefix: ")
buff.WriteString(cfg.LogPrefix)
buff.WriteString("\nWorkers: \n")
for _, w := range cfg.Workers {
buff.WriteString("---\n name: ")
buff.WriteString(w.Name)
buff.WriteString("\n repos: ")
for _, r := range w.Repos {
buff.WriteString("\n ")
buff.WriteString(r)
}
buff.WriteString("\n folder: ")
buff.WriteString(w.Folder)
buff.WriteString("\n logfile: ")
buff.WriteString(w.Logfile)
buff.WriteString("\n tagfile: ")
buff.WriteString(w.Tagfile)
buff.WriteString("\n keyrings: ")
for _, k := range w.Keyrings {
buff.WriteString("\n ")
buff.WriteString(k)
}
buff.WriteString("\n...\n")
}
return buff.String()
}

30
examples/create_spool_message.sh
Unescape View File

@ -0,0 +1,30 @@
#!/bin/sh
##function
write_message(){
orev=${3:-"a1b2c3d4e5f6"}
nrev=${4:-"9a8b7c6d5e4f"}
cat <<EOF
---
m_id: 123456
m_repo: $1
m_branch: $2
m_oldrev: $orev
m_newrev: $nrev
...
EOF
}
if [ $# -le 1 ]; then
echo "Usage: $0 <repo> <branch> [<oldrev> [<newrev]]"
exit 1
fi
write_message $@

29
examples/scorsh_example.cfg
Unescape View File

@ -0,0 +1,29 @@
---
s_spooldir: "./spool"
s_logfile: "./scorsh.log"
s_logprefix: "[scorsh]"
s_workers:
[
{
w_name: worker1,
w_repos: [".*:.*"], # All branches in all repos
w_folder: ./worker1,
w_logfile: ./worker1/worker1.log,
w_tagfile: "./worker1/worker1.cfg",
w_keyrings: [
"./worker1/allowed_users.asc"
]
},
{
w_name: worker2,
w_repos: [".*:master"], # Branch master in all repos
w_folder: ./worker2,
w_logfile: ./worker2/worker2.log,
w_tagfile: "./worker2/worker2.cfg",
w_keyrings: [
"./worker2/allowed_users.asc"
]
}
]
...

62
examples/worker1/allowed_users.asc
Unescape View File

@ -0,0 +1,62 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQGiBEFi1/MRBADTOYQBLugy99OG588zPBaOhDPaCfeeB/XiMbMLdO6RzCCZtuU7
e1G3I+8yIOLNUhfkmIT5Q7aU7FQA6OEexMvA3hijma7uLWs0GPGBC6U2XWEGVCcM
NNCVgZXv8JAEGdyWZmYBO+StYzp7tPhoujUMbY3ChPFes2IB1tlpJeYkuwCglKi6
ENT7n1pp0ZL02HyW7sUeFIED/3X1G6hKpcO12KXhdl70bI1ELBEoXW8S6E5+zN9v
bj/3SDVMMc99k7vmxd8MVhQviCuwHdX9115fiuUcb6atSdtbXMvCR729rlH+QfCA
aEdJ5O784zcpaTaplRlSVhqbkqU0O0qs2Uwpzyq2YOmqOWaUoxWjaAEZ3MTinJ05
FOIrA/4xN/kC0xJmqtAYg+IXnEM91pJaHVn1tlG0Us/ZUcV3qOBVzlxbELiYJY/P
f0RdSdJpsCglMeHMvKXYWDYeUwCxVnrX9QdY5U+o7jajW3CY+QXyiUOyB6Oxp1ZB
R9/Kzch8ZDG1efvhPS6Yl6c4VzrOEfmYfq0zA8dD81Q7fKoWKrQ0VmluY2Vuem8g
Tmljb3NpYSAoS2F0b2xhWikgPG1lQGthdG9sYXouaG9tZXVuaXgubmV0PoheBBMR
AgAeBQJBhMseAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEF8gs64LXwYv3+EA
n0DltQTOk4+jUcxj/EsAqlWRCeuwAJ0doTEepP8DZSP5CTdd6NFB1PdmzLQtVmlu
Y2Vuem8gTmljb3NpYSAoS2F0b2xhWikgPGthdG9sYXpAeWFob28uaXQ+iF4EExEC
AB4FAkFi1/MCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQXyCzrgtfBi+8mQCf
X/yJFPgwIwxh4DKNgaklCqp54sYAn0Y9gGbbx6GXgKq3HuUotw10uf5giEYEEBEC
AAYFAkFz0mYACgkQodsYiAfnHNnmGACfdDSOopfIhJ6QeoXO70s374CpTGMAn13h
Fo1L15WsRDGVmltPCyxgIkSWiEYEExECAAYFAkF0QPUACgkQAYe00nZD+a+ZSgCf
X/MNObMst8iZqavGJRQjWiHmJOUAn1SlDmC4El2Mv6UJB/MxZxDkoDcPiEYEExEC
AAYFAkF0CCkACgkQWDOBDtzNIgVcfgCgxIGF1+W/FNAavZ75fWBAgJXxdTcAoJFr
7rYllKXuaUbgzKRVgdO9JT9diEYEExECAAYFAkFz8ewACgkQnFc9aLrD67HS7wCg
nOXXrI+nJuMbyYIEejF2IOC0l/EAoKEfdFk5zx90P4qi/N0tyRsOARlmiEYEExEC
AAYFAkFzkv8ACgkQ6tyjHCMzLlp5DACfQLvi/Ob1x8Fs2YnRqSFlHvj7hh4AnAx2
plP6AOvDL7VpB54Y7aAVN/55iEYEExECAAYFAkFyyqwACgkQTSZ6jadyvDFJegCf
UBEzE6Rct4w4wANQhiAbm2RSwYkAnj+mfaUghdVj6LjgqQn8d5+VmzBZiEYEExEC
AAYFAkFyMxQACgkQ9QhEMx2jMUK/FgCgodHF1MRE0r/MLNwv1IIrxCpncrkAn3vC
sEmGt3B41mc40kfmj+Pi30zViEYEEBECAAYFAkF4EroACgkQLUrLvHBE1gFRuQCf
ahlJgXhfpIJ2esi1taT5NtNSlncAoJhYo47lMvkRCIx25RUUoHl5GHH9iEYEEBEC
AAYFAkIkxE0ACgkQ02jWMQa6YLzHoACeOTBEUZKQjNf4BHLzW3TXizFOBqoAoJ+w
do4hRB2tJFdI3i1aVGQIju8aiEYEExECAAYFAkLz1FAACgkQG+p9XIlFCSBHjQCe
OJehivpP+jhioDeBKsPcNfK/7PgAmwTcEAkHzPTEcQcvLAxGMmV4KWnsiEYEEBEC
AAYFAkLEIWYACgkQeL/ecPnD1vDS+ACgsBa09BSGMVppYWkbsQKs1JpaWYwAn33Z
Z/Z9FbqVguZwbdjaA13VH08oiF4EExECAB4FAkFi1/MCGwMGCwkIBwMCAxUCAwMW
AgECHgECF4AACgkQXyCzrgtfBi+8mQCfUk2WzM0p3pM+MVeILWmKVemvwzgAniY+
pALW1rL2IbIinW+1XtgIBa8NtDFWaW5jZW56byAoS2F0b2xhWikgTmljb3NpYSA8
a2F0b2xhekBmcmVha25ldC5vcmc+iGIEExECACIFAlhnsMwCGwMGCwkIBwMCBhUI
AgkKCwQWAgMBAh4BAheAAAoJEF8gs64LXwYvdukAnjKbYH4UvZKVpVRkEpYidnWF
7ecNAJ4gcHdh+tYoqBlIIyDj/6X+p4CaA7QeS2F0b2xhWiA8a2F0b2xhekBmcmVh
a25ldC5vcmc+iGAEExECACAFAkT0JDcCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIX
gAAKCRBfILOuC18GL2wlAJ9s3FnaoAgftFyzkpWXHbguXqc+2QCggZKrTK+Z6b30
3M5bpwkVPFO2tkm0I0Vuem8gTmljb3NpYSA8a2F0b2xhekBmcmVha25ldC5vcmc+
iGAEExECACAFAkT0Sl4CGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRBfILOu
C18GL9WmAJ9am98TX/t2THTAJlLrLqKC2+IKkgCfQBmOXi9B0rlucPfG5tc1ATop
Ice0LkVuem8gTmljb3NpYSAtLSBLYXRvbGFaIDxrYXRvbGF6QGZyZWFrbmV0Lm9y
Zz6IYAQTEQIAIAUCRPQmDAIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEF8g
s64LXwYvJXUAnAm9EXFMUqbKQHpPptu1bevyXGieAJ0QuIFhr4CqcLIBt6eEYDOk
/Abhb7kCDQRBYtf/EAgAkRtE9UbyNVoZgBmctRNn1rZGOGh0D0pg7f7DhoNZfsel
CvzYTb6NN+CK4TPFE820nfi86xu6rBxL0NBmXMuhEQImSLZ3J5RbHpc6k+dXu8G7
qbH8eWiee+vaebrMou4j5zJE5KZBeTa/IV0fGf9U9JxGMQvQfgPMiEEjMf4BpxCd
xyA4Y7MxfcNlTrsK0D2N9oO54L4OtBMyLQicj9vCGX9idXkstpFnu6XywrlFpzCM
t0j4DVTOFom4goYneTimoZvkhAmTsU9WUHdQF7bSEdzCLirD+eHwkq/EVk8x84tC
IxfzaRqRnPAD1OcCeoRqRbyJX7f5gEWqDUVGj9howwADBgf/RrEDF75RhVaqLbU0
99wGe4pY5YpeZ44J0fO6LY44nu/0amDQ6Ijb9Bx2h31+z+/90Fm2b3o/AVoVbkj3
D5qElFPLPJq3znaLeHVP3nV53qLYZqEgbkUFeyVTauavquy27Wrf7UQGZexGBjLb
ppJcsm27hswBZwDdkubiHiA5VcxJIhk2SyBgvjSiwLa9nVsPpp8P1PlGH7e8ijTk
ynF2rI4+P9tGkskagHPbs7gLSbpfHDiex/U3p1V9ry6OsoIKcrZAx5do5PQi7iaz
JGXmPPu/XM9XR1+Gj9vCoxg56AHgAE9RAX6SH99ECtRLiCVbwGpVj98A0LRy7Nf9
SMX7gYhJBBgRAgAJBQJBYtf/AhsMAAoJEF8gs64LXwYvwWkAnAnoHai6n3a3WnM1
zIolhmQMfsj3AJ4i/olraFAACc1BCJESK6dVFiSvoQ==
=aSn2
-----END PGP PUBLIC KEY BLOCK-----

BIN
examples/worker1/allowed_users.pgp
View File

Binary file not shown.

24
examples/worker1/worker1.cfg
Unescape View File

@ -0,0 +1,24 @@
#
# This is the typical worker configuration file. The file should be
# called "worker_config.cfg", and will be placed inside the worker
# directory. It defines the tags understood by the worker, with the
# corresponding list of commands
#
#
---
w_tags:
[
{
t_name: "LOG",
t_keyrings: ["allowed_users.asc"],
t_commands: [
{
c_url: "file:///home/katolaz/bin/scorsh_script.sh"
## c_hash: "12da324fb76s924acbce"
}
]
}
]
...

62
examples/worker2/allowed_users.asc
Unescape View File

@ -0,0 +1,62 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQGiBEFi1/MRBADTOYQBLugy99OG588zPBaOhDPaCfeeB/XiMbMLdO6RzCCZtuU7
e1G3I+8yIOLNUhfkmIT5Q7aU7FQA6OEexMvA3hijma7uLWs0GPGBC6U2XWEGVCcM
NNCVgZXv8JAEGdyWZmYBO+StYzp7tPhoujUMbY3ChPFes2IB1tlpJeYkuwCglKi6
ENT7n1pp0ZL02HyW7sUeFIED/3X1G6hKpcO12KXhdl70bI1ELBEoXW8S6E5+zN9v
bj/3SDVMMc99k7vmxd8MVhQviCuwHdX9115fiuUcb6atSdtbXMvCR729rlH+QfCA
aEdJ5O784zcpaTaplRlSVhqbkqU0O0qs2Uwpzyq2YOmqOWaUoxWjaAEZ3MTinJ05
FOIrA/4xN/kC0xJmqtAYg+IXnEM91pJaHVn1tlG0Us/ZUcV3qOBVzlxbELiYJY/P
f0RdSdJpsCglMeHMvKXYWDYeUwCxVnrX9QdY5U+o7jajW3CY+QXyiUOyB6Oxp1ZB
R9/Kzch8ZDG1efvhPS6Yl6c4VzrOEfmYfq0zA8dD81Q7fKoWKrQ0VmluY2Vuem8g
Tmljb3NpYSAoS2F0b2xhWikgPG1lQGthdG9sYXouaG9tZXVuaXgubmV0PoheBBMR
AgAeBQJBhMseAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEF8gs64LXwYv3+EA
n0DltQTOk4+jUcxj/EsAqlWRCeuwAJ0doTEepP8DZSP5CTdd6NFB1PdmzLQtVmlu
Y2Vuem8gTmljb3NpYSAoS2F0b2xhWikgPGthdG9sYXpAeWFob28uaXQ+iF4EExEC
AB4FAkFi1/MCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQXyCzrgtfBi+8mQCf
X/yJFPgwIwxh4DKNgaklCqp54sYAn0Y9gGbbx6GXgKq3HuUotw10uf5giEYEEBEC
AAYFAkFz0mYACgkQodsYiAfnHNnmGACfdDSOopfIhJ6QeoXO70s374CpTGMAn13h
Fo1L15WsRDGVmltPCyxgIkSWiEYEExECAAYFAkF0QPUACgkQAYe00nZD+a+ZSgCf
X/MNObMst8iZqavGJRQjWiHmJOUAn1SlDmC4El2Mv6UJB/MxZxDkoDcPiEYEExEC
AAYFAkF0CCkACgkQWDOBDtzNIgVcfgCgxIGF1+W/FNAavZ75fWBAgJXxdTcAoJFr
7rYllKXuaUbgzKRVgdO9JT9diEYEExECAAYFAkFz8ewACgkQnFc9aLrD67HS7wCg
nOXXrI+nJuMbyYIEejF2IOC0l/EAoKEfdFk5zx90P4qi/N0tyRsOARlmiEYEExEC
AAYFAkFzkv8ACgkQ6tyjHCMzLlp5DACfQLvi/Ob1x8Fs2YnRqSFlHvj7hh4AnAx2
plP6AOvDL7VpB54Y7aAVN/55iEYEExECAAYFAkFyyqwACgkQTSZ6jadyvDFJegCf
UBEzE6Rct4w4wANQhiAbm2RSwYkAnj+mfaUghdVj6LjgqQn8d5+VmzBZiEYEExEC
AAYFAkFyMxQACgkQ9QhEMx2jMUK/FgCgodHF1MRE0r/MLNwv1IIrxCpncrkAn3vC
sEmGt3B41mc40kfmj+Pi30zViEYEEBECAAYFAkF4EroACgkQLUrLvHBE1gFRuQCf
ahlJgXhfpIJ2esi1taT5NtNSlncAoJhYo47lMvkRCIx25RUUoHl5GHH9iEYEEBEC
AAYFAkIkxE0ACgkQ02jWMQa6YLzHoACeOTBEUZKQjNf4BHLzW3TXizFOBqoAoJ+w
do4hRB2tJFdI3i1aVGQIju8aiEYEExECAAYFAkLz1FAACgkQG+p9XIlFCSBHjQCe
OJehivpP+jhioDeBKsPcNfK/7PgAmwTcEAkHzPTEcQcvLAxGMmV4KWnsiEYEEBEC
AAYFAkLEIWYACgkQeL/ecPnD1vDS+ACgsBa09BSGMVppYWkbsQKs1JpaWYwAn33Z
Z/Z9FbqVguZwbdjaA13VH08oiF4EExECAB4FAkFi1/MCGwMGCwkIBwMCAxUCAwMW
AgECHgECF4AACgkQXyCzrgtfBi+8mQCfUk2WzM0p3pM+MVeILWmKVemvwzgAniY+
pALW1rL2IbIinW+1XtgIBa8NtDFWaW5jZW56byAoS2F0b2xhWikgTmljb3NpYSA8
a2F0b2xhekBmcmVha25ldC5vcmc+iGIEExECACIFAlhnsMwCGwMGCwkIBwMCBhUI
AgkKCwQWAgMBAh4BAheAAAoJEF8gs64LXwYvdukAnjKbYH4UvZKVpVRkEpYidnWF
7ecNAJ4gcHdh+tYoqBlIIyDj/6X+p4CaA7QeS2F0b2xhWiA8a2F0b2xhekBmcmVh
a25ldC5vcmc+iGAEExECACAFAkT0JDcCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIX
gAAKCRBfILOuC18GL2wlAJ9s3FnaoAgftFyzkpWXHbguXqc+2QCggZKrTK+Z6b30
3M5bpwkVPFO2tkm0I0Vuem8gTmljb3NpYSA8a2F0b2xhekBmcmVha25ldC5vcmc+
iGAEExECACAFAkT0Sl4CGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRBfILOu
C18GL9WmAJ9am98TX/t2THTAJlLrLqKC2+IKkgCfQBmOXi9B0rlucPfG5tc1ATop
Ice0LkVuem8gTmljb3NpYSAtLSBLYXRvbGFaIDxrYXRvbGF6QGZyZWFrbmV0Lm9y
Zz6IYAQTEQIAIAUCRPQmDAIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEF8g
s64LXwYvJXUAnAm9EXFMUqbKQHpPptu1bevyXGieAJ0QuIFhr4CqcLIBt6eEYDOk
/Abhb7kCDQRBYtf/EAgAkRtE9UbyNVoZgBmctRNn1rZGOGh0D0pg7f7DhoNZfsel
CvzYTb6NN+CK4TPFE820nfi86xu6rBxL0NBmXMuhEQImSLZ3J5RbHpc6k+dXu8G7
qbH8eWiee+vaebrMou4j5zJE5KZBeTa/IV0fGf9U9JxGMQvQfgPMiEEjMf4BpxCd
xyA4Y7MxfcNlTrsK0D2N9oO54L4OtBMyLQicj9vCGX9idXkstpFnu6XywrlFpzCM
t0j4DVTOFom4goYneTimoZvkhAmTsU9WUHdQF7bSEdzCLirD+eHwkq/EVk8x84tC
IxfzaRqRnPAD1OcCeoRqRbyJX7f5gEWqDUVGj9howwADBgf/RrEDF75RhVaqLbU0
99wGe4pY5YpeZ44J0fO6LY44nu/0amDQ6Ijb9Bx2h31+z+/90Fm2b3o/AVoVbkj3
D5qElFPLPJq3znaLeHVP3nV53qLYZqEgbkUFeyVTauavquy27Wrf7UQGZexGBjLb
ppJcsm27hswBZwDdkubiHiA5VcxJIhk2SyBgvjSiwLa9nVsPpp8P1PlGH7e8ijTk
ynF2rI4+P9tGkskagHPbs7gLSbpfHDiex/U3p1V9ry6OsoIKcrZAx5do5PQi7iaz
JGXmPPu/XM9XR1+Gj9vCoxg56AHgAE9RAX6SH99ECtRLiCVbwGpVj98A0LRy7Nf9
SMX7gYhJBBgRAgAJBQJBYtf/AhsMAAoJEF8gs64LXwYvwWkAnAnoHai6n3a3WnM1
zIolhmQMfsj3AJ4i/olraFAACc1BCJESK6dVFiSvoQ==
=aSn2
-----END PGP PUBLIC KEY BLOCK-----

BIN
examples/worker2/allowed_users.pgp
View File

Binary file not shown.

14
examples/worker2/worker2.cfg
Unescape View File

@ -0,0 +1,14 @@
---
w_tags:
[
{
t_name: "DEPLOY",
t_keyrings: ["allowed_users.asc"],
t_commands: [
{
c_url: "file:///home/katolaz/bin/deploy.sh"
}
]
}
]
...

44
sandpit/test_parse_message.go
Unescape View File

@ -0,0 +1,44 @@
package main
import(
"fmt"
"github.com/go-yaml/yaml"
"io/ioutil"
"log"
"os"
)
var orig_msg= `
---
m_id: 123456
m_repo: master
m_branch: test_branch
m_oldrev: a1b2c3d4e5f6
m_newrev: 9a8b7c6d5e4f
...
`
func main(){
var msg *SCORSHmsg
msg = new(SCORSHmsg)
fname := "spool/test_2"
data, err := ioutil.ReadFile(fname)
if err != nil {
log.Printf("Unable to open file: %s\n", fname)
os.Exit(1)
}
err = yaml.Unmarshal([]byte(data), msg)
if err != nil{
log.Printf("Error parsing message: %s", err)
}
fmt.Printf("%s\n", msg)
}

26
sandpit/test_regexp.go
Unescape View File

@ -0,0 +1,26 @@
package main
import(
"regexp"
"log"
)
func main (){
pattern := ".*"
str := "my_string"
matched, err := regexp.MatchString(pattern, str)
if err != nil {
log.Fatal("Error matching string: ", err)
}
if matched {
log.Printf("Yes! '%s' matched '%s'\n", str, pattern)
} else {
log.Printf("Bad luck!\n")
}
}

63
sandpit/test_worker_cfg.go
Unescape View File

@ -0,0 +1,63 @@
package main
import(
"fmt"
"github.com/go-yaml/yaml"
"log"
"strings"
)
var worker_cfg = `
---
w_tags:
[
{
t_name: "BUILD",
t_keyrings: ["build_keyring.asc", "general_keyring.asc"],
t_commands: [
{
c_url: "file:///home/user/bin/script.sh $1 $2",
c_hash: "12da324fb76s924acbce"
},
{
c_url: "http://my.server.net/call.pl?branch=$1"
}
]
},
{
t_name: "PUBLISH",
t_keyrings: ["web_developers.asc"],
t_commands: [
{
c_url: "file:///usr/local/bin/publish.py $repo $branch",
c_hash: "3234567898765432345678"
}
]
}
]
...
`
func main(){
var w *SCORSHworker
w = new(SCORSHworker)
sep := "\n---\n"
idx := strings.Index(worker_cfg, sep)
err := yaml.Unmarshal([]byte(worker_cfg[idx:]), w)
if err != nil{
log.Printf("Error parsing message: %s", err)
}
fmt.Printf("%s\n", w)
}

48
scorsh.cfg
Unescape View File

@ -6,42 +6,24 @@ s_logprefix: "[scorsh]"
s_workers: s_workers:
[ [
{ {
w_name: catchall, w_name: worker1,
w_repos: ["*:*"], # All branches in all repos w_repos: [".*:.*"], # All branches in all repos
w_folder: ./catchall, w_folder: ./worker1,
w_logfile: ./catchall/catchall.log, w_logfile: ./worker1/worker1.log,
w_tagfile: "./catchall/tags.cfg", w_tagfile: "./worker1/worker1.cfg",
w_keyrings: [ w_keyrings: [
"./catchall/catchall_keyring.asc" "./worker1/allowed_users.asc"
] ]
}, },
{
w_name: ascii,
w_repos: ["*:suites/ascii", # branch "suites/ascii" in all the repos
"*:suites/ascii-updates",
"*:suites/ascii-security"
],
w_folder: ./ascii,
w_logfile: ./worker_ascii.log,
w_tagfile: "./ascii/tags.cfg",
w_keyrings: [
"./ascii/ascii_keyring.asc"
]
},
{ {
w_name: ascii-side-branches , w_name: worker2,
w_repos: [ w_repos: [".*:master"], # Branch master in all repos
"*:suites/ascii-proposed", w_folder: ./worker2,
"*:suites/ascii-proposed-updates" w_logfile: ./worker2/worker2.log,
], w_tagfile: "./worker2/worker2.cfg",
w_folder: ./ascii-side-branches,
w_logfile: ./worker_ascii_side_branches.log,
w_tagfile: "./ascii-side-branches/tags.cfg",
w_keyrings: [ w_keyrings: [
"./ascii/ascii_keyring.asc", "./worker2/allowed_users.asc"
"./ascii/ascii_proposed_keyring.asc", ]
"./ascii/ascii_proposed_updates_keyring.asc" }
], ]
}
]
... ...

39
scorsh.go
Unescape View File

@ -6,7 +6,20 @@ import (
"log" "log"
) )
// manage debugging messages
const debug debugging = true
type debugging bool
func (d debugging) log(format string, args ...interface{}){
if d {
log.Printf(format, args...)
}
}
///////////
var conf_file = flag.String("c", "./scorsh.cfg", "Configuration file for SCORSH") var conf_file = flag.String("c", "./scorsh.cfg", "Configuration file for SCORSH")
@ -37,9 +50,10 @@ func FindMatchingWorkers(master *SCORSHmaster, msg *SCORSHmsg) []*SCORSHworker {
var ret []*SCORSHworker var ret []*SCORSHworker
for _,w := range master.Workers { for idx,w := range master.Workers {
if w.Matches(msg.repo, msg.branch) { if w.Matches(msg.Repo, msg.Branch) {
ret = append(ret, &w) debug.log("--- Worker: %s matches %s:%s\n", w.Name, msg.Repo, msg.Branch)
ret = append(ret, &(master.Workers[idx]))
} }
} }
return ret return ret
@ -55,12 +69,19 @@ func Master(master *SCORSHmaster) {
matching_workers = make([]*SCORSHworker, len(master.Workers)) matching_workers = make([]*SCORSHworker, len(master.Workers))
log.Println("[master] Master started ")
for { for {
select { select {
// - receive stuff from the spooler // - receive stuff from the spooler
case push_msg = <- master.Spooler: case push_msg = <- master.Spooler:
debug.log("[master] received message: %s\n", push_msg)
// - lookup the repos map for matching workers // - lookup the repos map for matching workers
matching_workers = FindMatchingWorkers(master, &push_msg) matching_workers = FindMatchingWorkers(master, &push_msg)
debug.log("[master] matching workers: %s\n", matching_workers)
// add the message to PendingMsg // add the message to PendingMsg
//... //...
// - dispatch the message to all the matching workers // - dispatch the message to all the matching workers
@ -75,11 +96,14 @@ func Master(master *SCORSHmaster) {
func InitMaster() *SCORSHmaster { func InitMaster() *SCORSHmaster {
master := ReadGlobalConfig(*conf_file) master := ReadGlobalConfig(*conf_file)
master.Repos = make(map[string][]*SCORSHworker) master.Repos = make(map[string][]*SCORSHworker)
master.WorkingMsg = make(map[string]int) master.WorkingMsg = make(map[string]int)
// This is the mutex-channel on which we receive acks from workers // This is the mutex-channel on which we receive acks from workers
master.StatusChan = make(chan SCORSHmsg, 1) master.StatusChan = make(chan SCORSHmsg, 1)
master.Spooler = make(chan SCORSHmsg, 1)
err_workers := StartWorkers(master) err_workers := StartWorkers(master)
if err_workers != nil { if err_workers != nil {
@ -90,9 +114,7 @@ func InitMaster() *SCORSHmaster {
err_spooler := StartSpooler(master) err_spooler := StartSpooler(master)
if err_spooler != nil { if err_spooler != nil {
log.Fatal("Error starting spooler: ", err_spooler) log.Fatal("Error starting spooler: ", err_spooler)
} else { }
log.Println("Spooler started correctly")
}
return master return master
} }
@ -101,8 +123,11 @@ func InitMaster() *SCORSHmaster {
func main() { func main() {
flag.Parse() flag.Parse()
master := InitMaster() master := InitMaster()
go Master(master) go Master(master)
<- master.StatusChan
} }

42
spooler.go
Unescape View File

@ -1,40 +1,59 @@
package main package main
import ( import (
"fmt"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"github.com/go-yaml/yaml"
"io/ioutil"
"log" "log"
"os" // "time"
"fmt"
) )
// parse a request file and return a SCORSHmessage // parse a request file and return a SCORSHmessage
func parse_request(fname string) (SCORSHmsg, error) { func parse_request(fname string, msg *SCORSHmsg) error {
var ret SCORSHmsg
_, err := os.Open(fname) debug.log("[parse_request] message at start: %s\n", msg)
data, err := ioutil.ReadFile(fname)
if err != nil { if err != nil {
log.Printf("Unable to open file: %s\n", fname) log.Printf("Unable to open file: %s\n", fname)
return ret, SCORSHerr(SCORSH_ERR_NO_FILE) return SCORSHerr(SCORSH_ERR_NO_FILE)
} }
// FIXME: Fill in the ret structure debug.log("[parse_request] file contains: \n%s\n", data)
debug.log("[parse_request] reading message from file: %s\n", fname)
return ret, nil
err = yaml.Unmarshal([]byte(data), msg)
if err != nil {
return fmt.Errorf("Error parsing message: %s", err)
}
debug.log("[parse_request] got message: %s\n", msg)
return nil
} }
func spooler(watcher *fsnotify.Watcher, worker chan SCORSHmsg) { func spooler(watcher *fsnotify.Watcher, worker chan SCORSHmsg) {
log.Println("Spooler started correctly")
var msg *SCORSHmsg
msg = new(SCORSHmsg)
for { for {
select { select {
case event := <-watcher.Events: case event := <-watcher.Events:
if event.Op == fsnotify.Create { if event.Op == fsnotify.Write {
msg, err := parse_request(event.Name) //time.Sleep(1000 * time.Millisecond)
debug.log("[spooler] new file %s detected\n", event.Name)
err := parse_request(event.Name, msg)
if err != nil { if err != nil {
log.Printf("Invalid packet received. [%s]\n", err) log.Printf("Invalid packet received. [%s]\n", err)
} }
worker <- msg debug.log("[spooler] read message: %s\n", msg)
worker <- *msg
} }
case err := <-watcher.Errors: case err := <-watcher.Errors:
log.Println("error:", err) log.Println("error:", err)
@ -59,5 +78,4 @@ func StartSpooler(master *SCORSHmaster) error {
go spooler(watcher, master.Spooler) go spooler(watcher, master.Spooler)
return nil return nil
} }

86
types.go
Unescape View File

@ -1,6 +1,7 @@
package main package main
import ( import (
"bytes"
"golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp"
) )
@ -15,11 +16,11 @@ const (
// the SCORSHmsg type represents messages received from the spool and // the SCORSHmsg type represents messages received from the spool and
// sent to workers // sent to workers
type SCORSHmsg struct { type SCORSHmsg struct {
name string Name string `yaml:"m_id"`
repo string Repo string `yaml:"m_repo"`
branch string Branch string `yaml:"m_branch"`
old_rev string Old_rev string `yaml:"m_oldrev"`
new_rev string New_rev string `yaml:"m_newrev"`
} }
type SCORSHcmd struct { type SCORSHcmd struct {
@ -35,17 +36,17 @@ type SCORSHtag struct {
// Configuration of a worker // Configuration of a worker
type SCORSHworker_cfg struct { type SCORSHworker_cfg struct {
Name string `yaml:"w_name"` Name string `yaml:"w_name"`
Repos []string `yaml:"w_repos"` Repos []string `yaml:"w_repos"`
Folder string `yaml:"w_folder"` Folder string `yaml:"w_folder"`
Logfile string `yaml:"w_logfile"` Logfile string `yaml:"w_logfile"`
Tagfile string `yaml:"w_tagfile"` Tagfile string `yaml:"w_tagfile"`
Keyrings []string `yaml:"w_keyrings"` Keyrings []string `yaml:"w_keyrings"`
Tags []SCORSHtag `yaml:"w_tags"`
} }
// State of a worker // State of a worker
type SCORSHworker_state struct { type SCORSHworker_state struct {
Tags []SCORSHtag `yaml:"w_tags"`
Keys map[string]openpgp.KeyRing Keys map[string]openpgp.KeyRing
MsgChan chan SCORSHmsg MsgChan chan SCORSHmsg
StatusChan chan SCORSHmsg StatusChan chan SCORSHmsg
@ -55,7 +56,7 @@ type SCORSHworker_state struct {
// worker // worker
type SCORSHworker struct { type SCORSHworker struct {
SCORSHworker_cfg `yaml:",inline"` SCORSHworker_cfg `yaml:",inline"`
SCORSHworker_state `yaml:",inline"` SCORSHworker_state
} }
// Configuration of the master // Configuration of the master
@ -80,3 +81,62 @@ type SCORSHmaster struct {
SCORSHmaster_cfg `yaml:",inline"` SCORSHmaster_cfg `yaml:",inline"`
SCORSHmaster_state SCORSHmaster_state
} }
func (cfg *SCORSHmaster) String() string {
var buff bytes.Buffer
buff.WriteString("spooldir: ")
buff.WriteString(cfg.Spooldir)
buff.WriteString("\nlogfile: ")
buff.WriteString(cfg.Logfile)
buff.WriteString("\nlogprefix: ")
buff.WriteString(cfg.LogPrefix)
buff.WriteString("\nWorkers: \n")
for _, w := range cfg.Workers {
buff.WriteString("---\n name: ")
buff.WriteString(w.Name)
buff.WriteString("\n repos: ")
for _, r := range w.Repos {
buff.WriteString("\n ")
buff.WriteString(r)
}
buff.WriteString("\n folder: ")
buff.WriteString(w.Folder)
buff.WriteString("\n logfile: ")
buff.WriteString(w.Logfile)
buff.WriteString("\n tagfile: ")
buff.WriteString(w.Tagfile)
buff.WriteString("\n keyrings: ")
for _, k := range w.Keyrings {
buff.WriteString("\n ")
buff.WriteString(k)
}
buff.WriteString("\n...\n")
}
return buff.String()
}
func (msg *SCORSHmsg) String() string {
var buff bytes.Buffer
buff.WriteString("\nName: ")
buff.WriteString(msg.Name)
buff.WriteString("\nRepo: ")
buff.WriteString(msg.Repo)
buff.WriteString("\nBranch: ")
buff.WriteString(msg.Branch)
buff.WriteString("\nOld_rev: ")
buff.WriteString(msg.Old_rev)
buff.WriteString("\nNew_rev: ")
buff.WriteString(msg.New_rev)
return buff.String()
}

46
workers.go
Unescape View File

@ -19,6 +19,8 @@ func (worker *SCORSHworker) Matches(repo, branch string) bool {
branch_pattern := parts[1] branch_pattern := parts[1]
repo_match, _ := regexp.MatchString(repo_pattern, repo) repo_match, _ := regexp.MatchString(repo_pattern, repo)
branch_match, _ := regexp.MatchString(branch_pattern, branch) branch_match, _ := regexp.MatchString(branch_pattern, branch)
debug.log("[worker.Matches] repo_match: %s\n", repo_match)
debug.log("[worker.Matches] branch_match: %s\n", branch_match)
if repo_match && branch_match { if repo_match && branch_match {
return true return true
} }
@ -59,34 +61,40 @@ func (w *SCORSHworker) LoadTags() error {
w_tags, err := ioutil.ReadFile(w.Tagfile) w_tags, err := ioutil.ReadFile(w.Tagfile)
if err != nil{ if err != nil{
log.Printf("[worker:%s] Cannot read worker config: ", w.Name, err) return fmt.Errorf("Cannot read worker config: %s", err)
return err
} }
err = yaml.Unmarshal(w_tags, w.Tags) err = yaml.Unmarshal(w_tags, w)
//err = yaml.Unmarshal(w_tags, tags)
if err != nil { if err != nil {
log.Printf("[worker:%s] Error while reading tags: ", w.Name, err) return fmt.Errorf("Error while reading tags: %s", err)
return err
} }
return nil return nil
} }
// FIXME--- STILL UNDER HEAVY WORK... // FIXME--- still needs some work...
func SCORSHWorker(w *SCORSHworker) { func Worker(w *SCORSHworker) {
var msg SCORSHmsg
log.Printf("[worker: %s] Started\n", w.Name)
w.StatusChan <- msg
// This is the main worker loop // This is the main worker loop
for { for {
select { select {
case msg := <-w.MsgChan: case msg = <-w.MsgChan:
debug.log("[worker: %s] received message %s\n", w.Name, msg.Name)
// process message // process message
err := walk_commits(msg, w) // err := walk_commits(msg, w)
if err != nil { // if err != nil {
log.Printf("[worker: %s] error in walk_commits: %s", err) // log.Printf("[worker: %s] error in walk_commits: %s", err)
} // }
log.Printf("[worker: %s] Received message: ", w.Name, msg)
} }
} }
} }
@ -99,7 +107,9 @@ func StartWorkers(master *SCORSHmaster) error {
// We should now start each worker // We should now start each worker
for w:=1; w<num_workers; w++ { log.Printf("num_workers: %d\n", num_workers)
for w:=0; w<num_workers; w++ {
worker := & (master.Workers[w]) worker := & (master.Workers[w])
// Set the Status and Msg channels // Set the Status and Msg channels
@ -108,21 +118,21 @@ func StartWorkers(master *SCORSHmaster) error {
// Load worker keyrings // Load worker keyrings
err := worker.LoadKeyrings() err := worker.LoadKeyrings()
if err != nil { if err != nil {
log.Printf("[worker: %s] Unable to load keyrings (Exiting): %s\n", worker.Name, err)
close(worker.MsgChan) close(worker.MsgChan)
return err return fmt.Errorf("[Starting worker: %s] Unable to load keyrings: %s\n", worker.Name, err)
} }
// Load worker tags from worker.Tagfile // Load worker tags from worker.Tagfile
err = worker.LoadTags() err = worker.LoadTags()
if err != nil { if err != nil {
log.Printf("[worker: %s] Unable to load tags (Exiting): %s\n", worker.Name, err)
close(worker.MsgChan) close(worker.MsgChan)
return err return fmt.Errorf("[Starting worker: %s] Unable to load tags: %s\n", worker.Name, err)
} }
// Add the repos definitions to the map master.Repos // Add the repos definitions to the map master.Repos
for _, repo_name := range worker.Repos { for _, repo_name := range worker.Repos {
master.Repos[repo_name] = append(master.Repos[repo_name], worker) master.Repos[repo_name] = append(master.Repos[repo_name], worker)
} }
go Worker(worker)
<- master.StatusChan
} }
return nil return nil
} }

Write Preview
Loading…
Cancel
Save