katolaz
/
scorsh
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

The pipeline "spooling -> routing -> file deletion" works

Browse Source
lint-refactor
KatolaZ 8 years ago
parent 74f8a74ecf
commit 7df9111d9f
5 changed files with 145 additions and 84 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 30
      commits.go
  2. 54
      scorsh.go
  3. 29
      spooler.go
  4. 83
      types.go
  5. 13
      workers.go

30
commits.go
Unescape View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"github.com/KatolaZ/git2go" "github.com/KatolaZ/git2go"
"golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp"
"log"
"os" "os"
"strings" "strings"
// "log" // "log"
@ -23,15 +24,15 @@ func CommitToString(commit *git.Commit) string {
} }
// FIXME: RETURN THE ENTITY PROVIDED BY THE CHECK, OR nil // FIXME: RETURN THE ENTITY PROVIDED BY THE CHECK, OR nil
func check_signature(commit *git.Commit, keys []*openpgp.KeyRing) (signature, signed string, err error) { func check_signature(commit *git.Commit, keys *map[string]openpgp.KeyRing) (signature, signed string, err error) {
signature, signed, err = commit.ExtractSignature() signature, signed, err = commit.ExtractSignature()
if err == nil { if err == nil {
for _, keyring := range keys { for _, keyring := range *keys {
_, err_sig := _, err_sig :=
openpgp.CheckArmoredDetachedSignature(*keyring, strings.NewReader(signed), openpgp.CheckArmoredDetachedSignature(keyring, strings.NewReader(signed),
strings.NewReader(signature)) strings.NewReader(signature))
if err_sig == nil { if err_sig == nil {
@ -45,6 +46,14 @@ func check_signature(commit *git.Commit, keys []*openpgp.KeyRing) (signature, si
return "", "", err return "", "", err
} }
func find_scorsh_message(commit *git.Commit) (string, error) {
msg := commit.RawMessage()
debug.log("[find_scorsg_msg] found message:\n %s\n", msg)
return msg, nil
}
// traverse all the commits between two references, looking for scorsh // traverse all the commits between two references, looking for scorsh
// commands // commands
// fixme: we don't have just one keyring here.... // fixme: we don't have just one keyring here....
@ -91,12 +100,17 @@ func walk_commits(msg SCORSHmsg, w *SCORSHworker) error {
// check if it can be verified by any of the keyrings associated // check if it can be verified by any of the keyrings associated
// with the scorsh-tag // with the scorsh-tag
//signature, signed, err := check_signature(commit, &keyring) // check if the commit contains a scorsh command
_, err = find_scorsh_message(commit)
//signature, signed, err := check_signature(commit, &w.Keys)
//_, _, err := check_signature(commit, w.keys) //_, _, err := check_signature(commit, w.keys)
//if err != nil { if err != nil {
// log.Printf("%s\n", SCORSHerr(SCORSH_ERR_SIGNATURE)) log.Printf("[worker: %s] %s\n", w.Name, SCORSHerr(SCORSH_ERR_SIGNATURE))
// } else {
//}
}
cur_commit = commit.Parent(0) cur_commit = commit.Parent(0)
} else { } else {
fmt.Printf("Commit %x not found!\n", cur_commit.Id()) fmt.Printf("Commit %x not found!\n", cur_commit.Id())

54
scorsh.go
Unescape View File

@ -62,32 +62,52 @@ func Master(master *SCORSHmaster) {
// master main loop: // master main loop:
var matching_workers []*SCORSHworker var matching_workers []*SCORSHworker
var push_msg SCORSHmsg
matching_workers = make([]*SCORSHworker, len(master.Workers)) matching_workers = make([]*SCORSHworker, len(master.Workers))
log.Println("[master] Master started ") log.Println("[master] Master started ")
debug.log("[master] StatusChan: %s\n", master.StatusChan)
for { for {
debug.log("[master] Receive loop...\n")
select { select {
// - receive stuff from the spooler case push_msg := <-master.Spooler:
case push_msg = <-master.Spooler: // here we manage the stuff we receive from the spooler
debug.log("[master] received message: %s\n", push_msg) debug.log("[master] received message: %s\n", push_msg)
// - lookup the repos map for matching workers // - lookup the repos map for matching workers
matching_workers = FindMatchingWorkers(master, &push_msg) matching_workers = FindMatchingWorkers(master, &push_msg)
debug.log("[master] matching workers: %s\n", matching_workers) debug.log("[master] matching workers: \n%s\n", matching_workers)
// add the message to PendingMsg // add the message to WorkingMsg, if it's not a duplicate!
//... if _, ok := master.WorkingMsg[push_msg.Id]; ok {
log.Printf("[master] detected duplicate message %s \n", push_msg.Id)
} else {
master.WorkingMsg[push_msg.Id] = 0
// - dispatch the message to all the matching workers // - dispatch the message to all the matching workers
for _, w := range matching_workers { for _, w := range matching_workers {
// increase the counter associated to the message debug.log("[master] sending msg to worker: %s\n", w.Name)
// send the message to the worker
w.MsgChan <- push_msg w.MsgChan <- push_msg
// increase the counter associated to the message
master.WorkingMsg[push_msg.Id] += 1
debug.log("[master] now WorkingMsg[%s] is: %d\n", push_msg.Id, master.WorkingMsg[push_msg.Id])
}
} }
case done_msg := <-master.StatusChan:
// Here we manage a status message from a worker
debug.log("[master] received message from StatusChan: %s\n", done_msg)
if _, ok := master.WorkingMsg[done_msg.Id]; ok && master.WorkingMsg[done_msg.Id] > 0 {
master.WorkingMsg[done_msg.Id] -= 1
if master.WorkingMsg[done_msg.Id] == 0 {
delete(master.WorkingMsg, done_msg.Id)
master.Spooler <- done_msg
} }
} else {
log.Printf("[master] received completion event for non-existing message name: %s\n", done_msg.Id)
} }
}
}
debug.log("[master] Exiting the for loop, for some mysterious reason...\n")
} }
func InitMaster() *SCORSHmaster { func InitMaster() *SCORSHmaster {
@ -96,9 +116,12 @@ func InitMaster() *SCORSHmaster {
master.Repos = make(map[string][]*SCORSHworker) master.Repos = make(map[string][]*SCORSHworker)
master.WorkingMsg = make(map[string]int) master.WorkingMsg = make(map[string]int)
// This is the mutex-channel on which we receive acks from workers // This is the channel on which we receive acks from workers
master.StatusChan = make(chan SCORSHmsg, 1) master.StatusChan = make(chan SCORSHmsg)
master.Spooler = make(chan SCORSHmsg, 1) // This is the channel on which we exchange messages with the spooler
master.Spooler = make(chan SCORSHmsg)
debug.log("[InitMaster] StatusChan: %s\n", master.StatusChan)
err_workers := StartWorkers(master) err_workers := StartWorkers(master)
if err_workers != nil { if err_workers != nil {
@ -111,17 +134,18 @@ func InitMaster() *SCORSHmaster {
log.Fatal("Error starting spooler: ", err_spooler) log.Fatal("Error starting spooler: ", err_spooler)
} }
return master return master
} }
func main() { func main() {
var done chan int
flag.Parse() flag.Parse()
master := InitMaster() master := InitMaster()
go Master(master) go Master(master)
<-master.StatusChan // wait indefinitely -- we should implement signal handling...
<-done
} }

29
spooler.go
Unescape View File

@ -6,6 +6,7 @@ import (
"github.com/go-yaml/yaml" "github.com/go-yaml/yaml"
"io/ioutil" "io/ioutil"
"log" "log"
"os"
// "time" // "time"
) )
@ -32,28 +33,40 @@ func parse_request(fname string, msg *SCORSHmsg) error {
return nil return nil
} }
func spooler(watcher *fsnotify.Watcher, worker chan SCORSHmsg) { func spooler(watcher *fsnotify.Watcher, master chan SCORSHmsg) {
log.Println("Spooler started correctly") log.Println("Spooler started correctly")
var msg *SCORSHmsg
msg = new(SCORSHmsg)
for { for {
select { select {
case event := <-watcher.Events: case event := <-watcher.Events:
// Here we manage genuine events from fsnotify. We catch the
// "Write" event, which should happen only when the file is
// created
if event.Op == fsnotify.Write { if event.Op == fsnotify.Write {
//time.Sleep(1000 * time.Millisecond) var msg SCORSHmsg
debug.log("[spooler] new file %s detected\n", event.Name) debug.log("[spooler] new file %s detected\n", event.Name)
err := parse_request(event.Name, msg) err := parse_request(event.Name, &msg)
if err != nil { if err != nil {
log.Printf("Invalid packet received. [%s]\n", err) log.Printf("Invalid packet received. [%s]\n", err)
} }
debug.log("[spooler] read message: %s\n", msg) debug.log("[spooler] read message: %s\n", msg)
worker <- *msg msg.Path = event.Name
master <- msg
} }
case err := <-watcher.Errors: case err := <-watcher.Errors:
log.Println("error:", err) // here we manage event errors
log.Println("[spooler] error: ", err)
case msg := <-master:
// Here we receive messages from the master about files to be
// removed
log.Printf("[spooler] received deletion request for: %s\n", msg.Path)
err := os.Remove(msg.Path)
if err != nil {
log.Printf("[spooler] error removing file: %s\n", err)
} else {
log.Printf("[spooler] file %s successfully removed\n", msg.Path)
}
} }
} }
} }

83
types.go
Unescape View File

@ -2,6 +2,7 @@ package main
import ( import (
"bytes" "bytes"
"fmt"
"golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp"
) )
@ -16,11 +17,12 @@ const (
// the SCORSHmsg type represents messages received from the spool and // the SCORSHmsg type represents messages received from the spool and
// sent to workers // sent to workers
type SCORSHmsg struct { type SCORSHmsg struct {
Name string `yaml:"m_id"` Id string `yaml:"m_id"`
Repo string `yaml:"m_repo"` Repo string `yaml:"m_repo"`
Branch string `yaml:"m_branch"` Branch string `yaml:"m_branch"`
Old_rev string `yaml:"m_oldrev"` Old_rev string `yaml:"m_oldrev"`
New_rev string `yaml:"m_newrev"` New_rev string `yaml:"m_newrev"`
Path string
} }
type SCORSHcmd struct { type SCORSHcmd struct {
@ -28,7 +30,7 @@ type SCORSHcmd struct {
Hash string `yaml:"c_hash"` Hash string `yaml:"c_hash"`
} }
type SCORSHtag struct { type SCORSHtag_cfg struct {
Name string `yaml:"t_name"` Name string `yaml:"t_name"`
Keyrings []string `yaml:"t_keyrings"` Keyrings []string `yaml:"t_keyrings"`
Commands []SCORSHcmd `yaml:"t_commands"` Commands []SCORSHcmd `yaml:"t_commands"`
@ -42,7 +44,7 @@ type SCORSHworker_cfg struct {
Logfile string `yaml:"w_logfile"` Logfile string `yaml:"w_logfile"`
Tagfile string `yaml:"w_tagfile"` Tagfile string `yaml:"w_tagfile"`
Keyrings []string `yaml:"w_keyrings"` Keyrings []string `yaml:"w_keyrings"`
Tags []SCORSHtag `yaml:"w_tags"` Tags []SCORSHtag_cfg `yaml:"w_tags"`
} }
// State of a worker // State of a worker
@ -82,39 +84,30 @@ type SCORSHmaster struct {
SCORSHmaster_state SCORSHmaster_state
} }
// client commands
type SCORSHtag struct {
Tag string `yaml:"s_tag"`
Args []string `yaml:"s_args"`
}
type SCORSHclient_msg struct {
Tags []SCORSHtag `yaml:"scorsh"`
}
////////////////////////
func (cfg *SCORSHmaster) String() string { func (cfg *SCORSHmaster) String() string {
var buff bytes.Buffer var buff bytes.Buffer
buff.WriteString("spooldir: ") fmt.Fprintf(&buff, "spooldir: %s\n", cfg.Spooldir)
buff.WriteString(cfg.Spooldir) fmt.Fprintf(&buff, "logfile: %s\n", cfg.Logfile)
buff.WriteString("\nlogfile: ") fmt.Fprintf(&buff, "logprefix: %s\n", cfg.LogPrefix)
buff.WriteString(cfg.Logfile) fmt.Fprintf(&buff, "Workers: \n")
buff.WriteString("\nlogprefix: ")
buff.WriteString(cfg.LogPrefix)
buff.WriteString("\nWorkers: \n")
for _, w := range cfg.Workers { for _, w := range cfg.Workers {
buff.WriteString("---\n name: ") fmt.Fprintf(&buff, "%s", &w)
buff.WriteString(w.Name)
buff.WriteString("\n repos: ")
for _, r := range w.Repos {
buff.WriteString("\n ")
buff.WriteString(r)
}
buff.WriteString("\n folder: ")
buff.WriteString(w.Folder)
buff.WriteString("\n logfile: ")
buff.WriteString(w.Logfile)
buff.WriteString("\n tagfile: ")
buff.WriteString(w.Tagfile)
buff.WriteString("\n keyrings: ")
for _, k := range w.Keyrings {
buff.WriteString("\n ")
buff.WriteString(k)
}
buff.WriteString("\n...\n")
} }
return buff.String() return buff.String()
@ -123,16 +116,26 @@ func (cfg *SCORSHmaster) String() string {
func (msg *SCORSHmsg) String() string { func (msg *SCORSHmsg) String() string {
var buff bytes.Buffer var buff bytes.Buffer
buff.WriteString("\nName: ") fmt.Fprintf(&buff, "Id: %s\n", msg.Id)
buff.WriteString(msg.Name) fmt.Fprintf(&buff, "Repo: %s\n", msg.Repo)
buff.WriteString("\nRepo: ") fmt.Fprintf(&buff, "Branch: %s\n", msg.Branch)
buff.WriteString(msg.Repo) fmt.Fprintf(&buff, "Old_Rev: %s\n", msg.Old_rev)
buff.WriteString("\nBranch: ") fmt.Fprintf(&buff, "New_rev: %s\n", msg.New_rev)
buff.WriteString(msg.Branch) fmt.Fprintf(&buff, "Path: %s\n", msg.Path)
buff.WriteString("\nOld_rev: ")
buff.WriteString(msg.Old_rev)
buff.WriteString("\nNew_rev: ")
buff.WriteString(msg.New_rev)
return buff.String() return buff.String()
} }
func (w *SCORSHworker) String() string {
var buff bytes.Buffer
fmt.Fprintf(&buff, "Name: %s\n", w.Name)
fmt.Fprintf(&buff, "Repos: %s\n", w.Repos)
fmt.Fprintf(&buff, "Folder: %s\n", w.Folder)
fmt.Fprintf(&buff, "Logfile: %s\n", w.Logfile)
fmt.Fprintf(&buff, "Tagfile: %s\n", w.Tagfile)
fmt.Fprintf(&buff, "Keyrings: %s\n", w.Keyrings)
return buff.String()
}

13
workers.go
Unescape View File

@ -9,6 +9,7 @@ import (
"os" "os"
"regexp" "regexp"
"strings" "strings"
"time"
) )
func (worker *SCORSHworker) Matches(repo, branch string) bool { func (worker *SCORSHworker) Matches(repo, branch string) bool {
@ -80,20 +81,26 @@ func Worker(w *SCORSHworker) {
var msg SCORSHmsg var msg SCORSHmsg
log.Printf("[worker: %s] Started\n", w.Name) log.Printf("[worker: %s] Started\n", w.Name)
debug.log("[worker: %s] MsgChan: %s\n", w.Name, w.MsgChan)
// notify that we have been started!
w.StatusChan <- msg w.StatusChan <- msg
// This is the main worker loop // This is the main worker loop
for { for {
select { select {
case msg = <-w.MsgChan: case msg = <-w.MsgChan:
debug.log("[worker: %s] received message %s\n", w.Name, msg.Name) debug.log("[worker: %s] received message %s\n", w.Name, msg.Id)
// process message // process message
// err := walk_commits(msg, w) // err := walk_commits(msg, w)
// if err != nil { // if err != nil {
// log.Printf("[worker: %s] error in walk_commits: %s", err) // log.Printf("[worker: %s] error in walk_commits: %s", err)
// } // }
log.Printf("[worker: %s] Received message: ", w.Name, msg) debug.log("[worker: %s] Received message: %s", w.Name, msg)
debug.log("[worker: %s] StatusChan: %s\n", w.Name, w.StatusChan)
time.Sleep(1000 * time.Millisecond)
w.StatusChan <- msg
debug.log("[worker: %s] Sent message back: %s", w.Name, msg)
} }
} }
} }
@ -113,7 +120,7 @@ func StartWorkers(master *SCORSHmaster) error {
worker := &(master.Workers[w]) worker := &(master.Workers[w])
// Set the Status and Msg channels // Set the Status and Msg channels
worker.StatusChan = master.StatusChan worker.StatusChan = master.StatusChan
worker.MsgChan = make(chan SCORSHmsg) worker.MsgChan = make(chan SCORSHmsg, 10)
// Load worker keyrings // Load worker keyrings
err := worker.LoadKeyrings() err := worker.LoadKeyrings()
if err != nil { if err != nil {

Write Preview
Loading…
Cancel
Save