Initial commit of rsync docker stuff.

Signed-off-by: Alec Warner <antarus@gentoo.org>

14 files changed, 263 insertions, 153 deletions

diff --git a/foundation.gentoo.org/golang/members/Makefile b/foundation.gentoo.org/golang/members/Makefile
deleted file mode 100644
index b050cf8..0000000
--- a/foundation.gentoo.org/golang/members/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-member.pb.go:
-	protoc --go_out ./ member.proto
-
-clean:
-	rm member.pb.go
diff --git a/foundation.gentoo.org/golang/members/list.go b/foundation.gentoo.org/golang/members/list.go
deleted file mode 100644
index c997f45..0000000
--- a/foundation.gentoo.org/golang/members/list.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package members
-
-import (
-  "flag"
-  "fmt"
-  "os"
-  "strings"
-
-  "github.com/google/subcommands"
-  "golang.org/x/net/context"
-)
-
-type listCmd struct {
-  membersListPath string
-}
-
-func (*listCmd) Name() string     { return "list" }
-func (*listCmd) Synopsis() string { return "list members to stdout." }
-func (*listCmd) Usage() string {
-  return `print --membersListPath /path/to/members`
-}
-
-func (p *listCmd) SetFlags(f *flag.FlagSet) {
-   f.StringVar(&p.membersListPath, "membersListPath", './members', "path to members list")
-}
-
-func (p *listCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
-  for _, arg := range f.Args() {
-    fmt.Printf("%s ", arg)
-  }
-  fmt.Println()
-  return subcommands.ExitSuccess
-}
diff --git a/foundation.gentoo.org/golang/members/main.go b/foundation.gentoo.org/golang/members/main.go
deleted file mode 100644
index c726053..0000000
--- a/foundation.gentoo.org/golang/members/main.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package members
-
-import (
-  "flag"
-  "os"
-  "strings"
-
-  "github.com/google/subcommands"
-  "golang.org/x/net/context"
-)
-
-
-func main() {
-  subcommands.Register(subcommands.HelpCommand(), "")
-  subcommands.Register(subcommands.FlagsCommand(), "")
-  subcommands.Register(subcommands.CommandsCommand(), "")
-  subcommands.Register(&list.ListCmd{}, "")
-
-  flag.Parse()
-  ctx := context.Background()
-  os.Exit(int(subcommands.Execute(ctx)))
-}
diff --git a/foundation.gentoo.org/golang/members/member.proto b/foundation.gentoo.org/golang/members/member.proto
deleted file mode 100644
index 22969af..0000000
--- a/foundation.gentoo.org/golang/members/member.proto
+++ /dev/null
@@ -1,16 +0,0 @@
-message Member {
-  // Unique idenfier for each member.
-  optional int64 id = 1;
-
-  // Email information
-  repeated string email = 2;
-
-  // Name
-  repeated string name = 3;
-
-  // microsecond timestamp of joining.
-  optional int64 join_timestamp_us = 4;
-
-  // microsecond timestamp of any elections member participated in
-  repeated int64 voted_in_election = 5;
-}
diff --git a/sites/foundation_tracker/Makefile b/sites/foundation_tracker/Makefile
deleted file mode 100644
index b050cf8..0000000
--- a/sites/foundation_tracker/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-member.pb.go:
-	protoc --go_out ./ member.proto
-
-clean:
-	rm member.pb.go
diff --git a/sites/foundation_tracker/member.proto b/sites/foundation_tracker/member.proto
deleted file mode 100644
index 22969af..0000000
--- a/sites/foundation_tracker/member.proto
+++ /dev/null
@@ -1,16 +0,0 @@
-message Member {
-  // Unique idenfier for each member.
-  optional int64 id = 1;
-
-  // Email information
-  repeated string email = 2;
-
-  // Name
-  repeated string name = 3;
-
-  // microsecond timestamp of joining.
-  optional int64 join_timestamp_us = 4;
-
-  // microsecond timestamp of any elections member participated in
-  repeated int64 voted_in_election = 5;
-}
diff --git a/src/foundation.gentoo.org/golang/members/data/member.pb.go b/src/foundation.gentoo.org/golang/members/data/member.pb.go
index 3d39f40..4f12ab6 100644
--- a/src/foundation.gentoo.org/golang/members/data/member.pb.go
+++ b/src/foundation.gentoo.org/golang/members/data/member.pb.go
@@ -11,6 +11,7 @@ It is generated from these files:
 It has these top-level messages:
 	MemberRoll
 	Member
+	Status
 */
 package data
 
@@ -21,6 +22,41 @@ import math "math"
 var _ = proto.Marshal
 var _ = math.Inf
 
+type Status_State int32
+
+const (
+	// Member has voting rights
+	Status_ACTIVE Status_State = 0
+	// Member had voting rights, but is no longer a foundation member.
+	Status_EMERITUS Status_State = 1
+)
+
+var Status_State_name = map[int32]string{
+	0: "ACTIVE",
+	1: "EMERITUS",
+}
+var Status_State_value = map[string]int32{
+	"ACTIVE":   0,
+	"EMERITUS": 1,
+}
+
+func (x Status_State) Enum() *Status_State {
+	p := new(Status_State)
+	*p = x
+	return p
+}
+func (x Status_State) String() string {
+	return proto.EnumName(Status_State_name, int32(x))
+}
+func (x *Status_State) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(Status_State_value, data, "Status_State")
+	if err != nil {
+		return err
+	}
+	*x = Status_State(value)
+	return nil
+}
+
 type MemberRoll struct {
 	// When we add a new member, they should receive this ID.
 	// We rely on clients to increment this ID during add operations.
@@ -55,11 +91,16 @@ type Member struct {
 	Email []string `protobuf:"bytes,2,rep,name=email" json:"email,omitempty"`
 	// Name
 	Name []string `protobuf:"bytes,3,rep,name=name" json:"name,omitempty"`
-	// microsecond timestamp of joining.
-	JoinTimestampUs *int64 `protobuf:"varint,4,opt,name=join_timestamp_us" json:"join_timestamp_us,omitempty"`
 	// microsecond timestamp of any elections member participated in
-	VotedInElection  []int64 `protobuf:"varint,5,rep,name=voted_in_election" json:"voted_in_election,omitempty"`
-	XXX_unrecognized []byte  `json:"-"`
+	VotedInElection []int64 `protobuf:"varint,4,rep,name=voted_in_election" json:"voted_in_election,omitempty"`
+	// Status's of a given member. Typically most users have 1 state
+	// (e.g. they are an active member of the foundation.) But we will
+	// try to record all state changes here, so if people leave and come
+	// back we have a record.
+	State []*Status `protobuf:"bytes,5,rep,name=state" json:"state,omitempty"`
+	// A list of gpg key bytes for each member.
+	Gpgkey           [][]byte `protobuf:"bytes,6,rep,name=gpgkey" json:"gpgkey,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
 }
 
 func (m *Member) Reset()         { *m = Member{} }
@@ -87,19 +128,55 @@ func (m *Member) GetName() []string {
 	return nil
 }
 
-func (m *Member) GetJoinTimestampUs() int64 {
-	if m != nil && m.JoinTimestampUs != nil {
-		return *m.JoinTimestampUs
+func (m *Member) GetVotedInElection() []int64 {
+	if m != nil {
+		return m.VotedInElection
 	}
-	return 0
+	return nil
 }
 
-func (m *Member) GetVotedInElection() []int64 {
+func (m *Member) GetState() []*Status {
 	if m != nil {
-		return m.VotedInElection
+		return m.State
 	}
 	return nil
 }
 
+func (m *Member) GetGpgkey() [][]byte {
+	if m != nil {
+		return m.Gpgkey
+	}
+	return nil
+}
+
+type Status struct {
+	// State of the member.
+	State *Status_State `protobuf:"varint,1,opt,name=state,enum=data.Status_State,def=0" json:"state,omitempty"`
+	// Time when state change occured.
+	StatusChange     *int64 `protobuf:"varint,2,opt,name=status_change" json:"status_change,omitempty"`
+	XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *Status) Reset()         { *m = Status{} }
+func (m *Status) String() string { return proto.CompactTextString(m) }
+func (*Status) ProtoMessage()    {}
+
+const Default_Status_State Status_State = Status_ACTIVE
+
+func (m *Status) GetState() Status_State {
+	if m != nil && m.State != nil {
+		return *m.State
+	}
+	return Default_Status_State
+}
+
+func (m *Status) GetStatusChange() int64 {
+	if m != nil && m.StatusChange != nil {
+		return *m.StatusChange
+	}
+	return 0
+}
+
 func init() {
+	proto.RegisterEnum("data.Status_State", Status_State_name, Status_State_value)
 }
diff --git a/src/foundation.gentoo.org/golang/members/data/member.proto b/src/foundation.gentoo.org/golang/members/data/member.proto
index 2e35208..500364e 100644
--- a/src/foundation.gentoo.org/golang/members/data/member.proto
+++ b/src/foundation.gentoo.org/golang/members/data/member.proto
@@ -18,9 +18,29 @@ message Member {
   // Name
   repeated string name = 3;
 
-  // microsecond timestamp of joining.
-  optional int64 join_timestamp_us = 4;
-
   // microsecond timestamp of any elections member participated in
-  repeated int64 voted_in_election = 5;
+  repeated int64 voted_in_election = 4;
+
+  // Status's of a given member. Typically most users have 1 state
+  // (e.g. they are an active member of the foundation.) But we will
+  // try to record all state changes here, so if people leave and come
+  // back we have a record.
+  repeated Status state = 5;
+
+  // A list of gpg key bytes for each member.
+  repeated bytes gpgkey = 6;
+}
+
+message Status {
+  enum State {
+    // Member has voting rights
+    ACTIVE = 0;
+    // Member had voting rights, but is no longer a foundation member.
+    EMERITUS = 1;
+  }
+
+  // State of the member.
+  optional State state = 1 [default=ACTIVE];
+  // Time when state change occured.
+  optional int64 status_change = 2;
 }
diff --git a/src/foundation.gentoo.org/golang/members/importers/ldap.go b/src/foundation.gentoo.org/golang/members/importers/ldap.go
index 02db9b0..02cc7eb 100644
--- a/src/foundation.gentoo.org/golang/members/importers/ldap.go
+++ b/src/foundation.gentoo.org/golang/members/importers/ldap.go
@@ -1,9 +1,3 @@
-package ldap
+package importers
 
-import (
-  "gopkg.in/ldap.v2"
-)
-
-func listDevs(ldap.v2.Conn) {
-
-}
+import ()
diff --git a/src/foundation.gentoo.org/golang/members/main.go b/src/foundation.gentoo.org/golang/members/main.go
deleted file mode 100644
index 18e7f65..0000000
--- a/src/foundation.gentoo.org/golang/members/main.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package main
-
-import (
-  "flag"
-  "os"
-
-  "foundation.gentoo.org/golang/members/maintenance"
-  "github.com/google/subcommands"
-  "golang.org/x/net/context"
-)
-
-
-func main() {
-  subcommands.Register(subcommands.HelpCommand(), "")
-  subcommands.Register(subcommands.FlagsCommand(), "")
-  subcommands.Register(subcommands.CommandsCommand(), "")
-  subcommands.Register(&maintenance.ListCmd{}, "")
-
-  flag.Parse()
-  ctx := context.Background()
-  os.Exit(int(subcommands.Execute(ctx)))
-}
diff --git a/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go b/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go
index 05017ea..12369d5 100644
--- a/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go
+++ b/src/foundation.gentoo.org/golang/members/maintenance/memberroll.go
@@ -1,20 +1,22 @@
 package maintenance
 
 import (
-  "io/ioutil"
+	"fmt"
+	"io/ioutil"
+	"time"
 
-  "github.com/golang/protobuf/proto"
-  pb "foundation.gentoo.org/golang/members/data"
+	pb "foundation.gentoo.org/golang/members/data"
+	"github.com/golang/protobuf/proto"
 )
 
 // Struct MemberRoll defines operations on a MemberRoll
 // and holds the underlying data.
 type MemberRoll struct {
-  roll *pb.MemberRoll
+	roll *pb.MemberRoll
 }
 
 // Create a new memberRoll from a file.
-func NewMemberRoll (path string) (*MemberRoll, error) {
+func NewMemberRoll(path string) (*MemberRoll, error) {
 	in, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, err
@@ -30,10 +32,10 @@ func NewMemberRoll (path string) (*MemberRoll, error) {
 }
 
 // NewEmptyMemberRoll creates an empty MemberRoll
-func NewEmptyMemberRoll() (*MemberRoll) {
+func NewEmptyMemberRoll() *MemberRoll {
 	// The first member gets ID 1.
 	return &MemberRoll{
-		roll: &pb.MemberRoll {
+		roll: &pb.MemberRoll{
 			NextMemberId: proto.Int64(1),
 		},
 	}
@@ -42,18 +44,71 @@ func NewEmptyMemberRoll() (*MemberRoll) {
 // Save will write the MemberRoll to file at path.
 func (mr *MemberRoll) Save(path string) (bool, error) {
 	bytes, err := proto.Marshal(mr.roll)
-	if err != nil { return false, err }
+	if err != nil {
+		return false, err
+	}
 	if err = ioutil.WriteFile(path, bytes, 0644); err != nil {
 		return false, err
 	}
 	return true, nil
 }
 
+// Does this email match an existing member?
+func (mr *MemberRoll) MatchEmail(email string) *pb.Member {
+	member := &pb.Member{
+		Email: []string{email},
+	}
+	return mr.Match(member)
+}
+
+// Members match if they shared an email address.
+func Match(m1, m2 *pb.Member) bool {
+	for _, i := range m1.GetEmail() {
+		for _, j := range m2.GetEmail() {
+			if i == j {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Match will try to find if m is in mr.
+func (mr *MemberRoll) Match(m *pb.Member) *pb.Member {
+	for _, i := range mr.roll.GetMembers() {
+		if Match(m, i) {
+			return i
+		}
+	}
+	return nil
+}
+
+// low-level function for add raw member protobufs to the roll.
+func (mr *MemberRoll) Add(m pb.Member) (bool, error) {
+	match := mr.Match(&m)
+	if match != nil {
+		err := fmt.Errorf("Adding existing member: %v", match)
+		return false, err
+	}
+	mr.roll.Members = append(mr.roll.Members, &m)
+	return true, nil
+}
+
+func (mr *MemberRoll) RecordVote(id int64, t time.Time) error {
+	return nil
+}
+
+func (mr *MemberRoll) Print() {
+	for _, member := range mr.roll.GetMembers() {
+		fmt.Printf("Member: %v\n", member)
+	}
+}
+
 // GetMaxMemberId computes the largest ID amongst a memberroll.
 // This works by assuming members are not removed from a roll.
-func (mr *MemberRoll) GetMaxMemberId() (int64) {
-	var id int64 = 0;
-	for _, member := range (mr.roll.GetMembers()) {
+func (mr *MemberRoll) GetMaxMemberId() int64 {
+	var id int64 = 0
+	for _, member := range mr.roll.GetMembers() {
 		member_id := member.GetId()
 		if member_id > id {
 			id = member_id
@@ -62,7 +117,7 @@ func (mr *MemberRoll) GetMaxMemberId() (int64) {
 	return id
 }
 
-func (mr *MemberRoll) GetNextMemberId() (int64) {
+func (mr *MemberRoll) GetNextMemberId() int64 {
 	id := mr.GetNextMemberId()
 	// If the member roll lost its ID (not a required field.)
 	// Recalculate based on existing membership.
diff --git a/src/infra.gentoo.org/rsync-node/Dockerfile b/src/infra.gentoo.org/rsync-node/Dockerfile
new file mode 100644
index 0000000..3530c25
--- /dev/null
+++ b/src/infra.gentoo.org/rsync-node/Dockerfile
@@ -0,0 +1,21 @@
+# image is based on stage3-amd64
+FROM gentoo/stage3-amd64:latest
+COPY rsyncd.conf /opt/rsync/rsyncd.conf
+COPY wrap_rsync.sh /opt/rsync/wrap_rsync.sh
+
+# Normally I would advocate for ARG here and pass arguments to wrap_rsync.
+# This would enable new docker builds with arguments like:
+# docker build . --build_arg WAIT_TIME=30m -t gentoo/rsync
+# However, ARG's cannot be passed to ENTRYPOINTs, so we set these as ENV instead.
+# Mirror to get data from.
+ENV SOURCE_MIRROR=rsync://boobie.gentoo.org/gentoo-portage
+# ENV SOURCE_MIRROR=rsync://rsync.us.gentoo.org/gentoo-portage
+# Where to write the data in this container.
+ENV DEST_DIR=/dev/shm/gentoo
+# How long to wait between syncs; must be a valid argument to sleep
+ENV WAIT_TIME=10m
+
+# Expose Rsync port
+EXPOSE 873
+
+ENTRYPOINT /opt/rsync/wrap_rsync.sh
diff --git a/src/infra.gentoo.org/rsync-node/rsyncd.conf b/src/infra.gentoo.org/rsync-node/rsyncd.conf
new file mode 100644
index 0000000..7fbd4b9
--- /dev/null
+++ b/src/infra.gentoo.org/rsync-node/rsyncd.conf
@@ -0,0 +1,19 @@
+
+uid = nobody
+gid = nobody
+# We are in a container, who cares.
+# use chroot = no
+# Let clients use as much as they want; CPU control is in a load balancer in front of us.
+max connections = 0
+
+motd file = ./rsyncd.motd
+log file = /var/log/rsync.log
+transfer logging = yes
+log format = %t %a %m %f %b
+syslog facility = local3
+timeout = 300
+  
+[gentoo-portage]
+path = /dev/shm/gentoo
+comment = Gentoo Linux Portage tree mirror
+exclude = distfiles
diff --git a/src/infra.gentoo.org/rsync-node/wrap_rsync.sh b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh
new file mode 100755
index 0000000..f92934c
--- /dev/null
+++ b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# On container start, run an rsync to get a good copy of the tree.
+# Then execute rsyncd; we will start serving once the sync completes.
+# Then keep syncing in the background every 30m.
+
+function sync() {
+	OPTS=(
+	 --quiet
+	 --recursive
+	 --links
+	 --perms
+	 --times
+	 --delete
+	 --timeout=300
+	 --checksum
+	)
+	SRC="${2}"
+	DST="${1}"
+  
+	echo "Started update at" $(date) >> $0.log 2>&1
+	logger -t rsync "re-rsyncing the gentoo-portage tree"
+	/usr/bin/rsync ${OPTS[@]} "${SRC}" "${DST}" >> $0.log 2>&1
+	echo "End: "$(date) >> $0.log 2>&1
+}
+
+sync "${DEST_DIR}/serving" "${SOURCE_MIRROR}" # this is synchronous.
+
+# Then launch rsyncd; it will detach into the background.
+rsync --daemon --config="/opt/rsync/rsyncd.conf"
+
+while true
+do
+				sleep "${WAIT_TIME}"
+				tmp=$(mktemp -d -p "${DEST_DIR}" XXXXXX)
+				# If we fail to sync, just try again.
+				if ! sync "${tmp}" "${SOURCE_MIRROR}"; then
+					rm -rf "${tmp}"
+					continue
+				fi
+				# Atomically rename
+				mv -f "${tmp}" "${DEST_DIR}/serving"
+done