From ece500c342b6acb78b8c5f6a524c57cb69977244 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Mon, 20 Mar 2023 00:54:42 +0100 Subject: [PATCH 01/20] Cloudflare Tunnel Support (#29) * Adding necessary changes for tunnel setup supporting cloudflared * Applied shwrap pattern to login, create and delete tunnel actions, many fixes * Refactoring sesh 1 * Added utils, remaining tunnel tasks, cleanup --- .gitignore | 3 + Makefile | 9 +- internal/system/system.go | 201 +++++++++++++++++++++++++++ internal/tasks/tasks.go | 175 +++++++++++++++++++++-- internal/utils/utils.go | 44 ++++++ scripts/cloudflared_login.sh | 6 + scripts/cloudflared_tunnel_create.sh | 7 + scripts/cloudflared_tunnel_delete.sh | 7 + 8 files changed, 434 insertions(+), 18 deletions(-) create mode 100644 scripts/cloudflared_login.sh create mode 100755 scripts/cloudflared_tunnel_create.sh create mode 100755 scripts/cloudflared_tunnel_delete.sh diff --git a/.gitignore b/.gitignore index d571598..964bc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ main # Build /bin + +# Logs from executed scripts +/scripts/output.log diff --git a/Makefile b/Makefile index f8be198..c09a58b 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ test-with-coverage: go test -tags=unit -timeout=600s -v ./... -coverprofile=coverage.out install-cloud: build-cloud + systemctl stop edgeboxctl cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl cp ./edgeboxctl/edgeboxctl.service /lib/systemd/system/edgeboxctl.service systemctl daemon-reload @@ -42,8 +43,10 @@ install-cloud: build-cloud @echo "To start edgeboxctl run: systemctl start edgeboxctl" install-prod: build-prod - cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl - cp ./edgeboxctl/edgeboxctl.service /lib/systemd/system/edgeboxctl.service - systemctl daemon-reload + -sudo systemctl stop edgeboxctl + sudo rm -rf /usr/local/bin/edgeboxctl /lib/systemd/system/edgeboxctl.service + sudo cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl + sudo cp ./edgeboxctl.service /lib/systemd/system/edgeboxctl.service + sudo systemctl daemon-reload @echo "Edgeboxctl installed successfully" @echo "To start edgeboxctl run: systemctl start edgeboxctl" \ No newline at end of file diff --git a/internal/system/system.go b/internal/system/system.go index a8684e8..95a6538 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -4,6 +4,13 @@ import ( "fmt" "strconv" "strings" + "log" + "os" + "os/exec" + "bufio" + "path/filepath" + "io/ioutil" + "encoding/json" "github.com/edgebox-iot/edgeboxctl/internal/utils" @@ -11,6 +18,12 @@ import ( "github.com/shirou/gopsutil/host" ) +type cloudflaredTunnelJson struct { + AccountTag string `json:"AccountTag"` + TunnelSecret string `json:"TunnelSecret"` + TunnelID string `json:"TunnelID"` +} + // GetUptimeInSeconds: Returns a value (as string) of the total system uptime func GetUptimeInSeconds() string { uptime, _ := host.Uptime() @@ -85,3 +98,191 @@ func SetupCloudOptions() { // In the end of this operation takes place, remove the env file as to not overwrite any options once they are set. utils.Exec("/", "rm", []string{cloudEnvFileLocationPath}) } + +// StartService: Starts a service +func StartService(serviceID string) { + wsPath := utils.GetPath(utils.WsPath) + fmt.Println("Starting" + serviceID + "service") + cmdargs := []string{"start", serviceID} + utils.Exec(wsPath, "systemctl", cmdargs) +} + +// StopService: Stops a service +func StopService(serviceID string) { + wsPath := utils.GetPath(utils.WsPath) + fmt.Println("Stopping" + serviceID + "service") + cmdargs := []string{"stop", "cloudflared"} + utils.Exec(wsPath, "systemctl", cmdargs) +} + +// RestartService: Restarts a service +func RestartService(serviceID string) { + wsPath := utils.GetPath(utils.WsPath) + fmt.Println("Restarting" + serviceID + "service") + cmdargs := []string{"restart", serviceID} + utils.Exec(wsPath, "systemctl", cmdargs) +} + +// GetServiceStatus: Returns the status output of a service +func GetServiceStatus(serviceID string) string { + wsPath := utils.GetPath(utils.WsPath) + cmdargs := []string{"status", serviceID} + return utils.Exec(wsPath, "systemctl", cmdargs) +} + +// CreateTunnel: Creates a tunnel via cloudflared, needs to be authenticated first +func CreateTunnel(configDestination string) { + fmt.Println("Creating Tunnel for Edgebox.") + cmd := exec.Command("sh", "/home/system/components/edgeboxctl/scripts/cloudflared_tunnel_create.sh") + stdout, err := cmd.StdoutPipe() + if err != nil { + panic(err) + } + scanner := bufio.NewScanner(stdout) + err = cmd.Start() + if err != nil { + panic(err) + } + for scanner.Scan() { + fmt.Println(scanner.Text()) + text := scanner.Text() + fmt.Println(text) + } + if scanner.Err() != nil { + cmd.Process.Kill() + cmd.Wait() + panic(scanner.Err()) + } + + // This also needs to be executed in root and non root variants + fmt.Println("Reading cloudflared folder to get the JSON file.") + isRoot := false + dir := "/home/system/.cloudflared/" + dir2 := "/root/.cloudflared/" + files, err := os.ReadDir(dir) + if err != nil { + panic(err) + } + + var jsonFile os.DirEntry + for _, file := range files { + // check if file has json extension + if filepath.Ext(file.Name()) == ".json" { + fmt.Println("Non-Root JSON file found: " + file.Name()) + jsonFile = file + } + } + + // If the files are not in the home folder, try the root folder + if jsonFile == nil { + files, err = os.ReadDir(dir2) + if err != nil { + panic(err) + } + for _, file := range files { + // check if file has json extension + if filepath.Ext(file.Name()) == ".json" { + fmt.Println("Root JSON file found: " + file.Name()) + jsonFile = file + isRoot = true + } + } + } + + if jsonFile == nil { + panic("No JSON file found in directory") + } + + fmt.Println("Reading JSON file.") + targetDir := "/home/system/.cloudflared/" + if isRoot { + targetDir = "/root/.cloudflared/" + } + + jsonFilePath := filepath.Join(targetDir, jsonFile.Name()) + jsonBytes, err := ioutil.ReadFile(jsonFilePath) + if err != nil { + panic(err) + } + + fmt.Println("Parsing JSON file.") + var data cloudflaredTunnelJson + err = json.Unmarshal(jsonBytes, &data) + if err != nil { + log.Printf("Error reading tunnel JSON file: %s", err) + } + + fmt.Println("Tunnel ID is:" + data.TunnelID) + + // create the config.yml file with the following content in each line: + // "url": "http://localhost:80" + // "tunnel": "" + // "credentials-file": "/root/.cloudflared/.json" + + file := configDestination + f, err := os.Create(file) + if err != nil { + panic(err) + } + + defer f.Close() + + _, err = f.WriteString("url: http://localhost:80\ntunnel: " + data.TunnelID + "\ncredentials-file: " + jsonFilePath) + + if err != nil { + panic(err) + } +} + +// DeleteTunnel: Deletes a tunnel via cloudflared, this does not remove the service +func DeleteTunnel() { + fmt.Println("Deleting possible previous tunnel.") + + // Configure the service and start it + cmd := exec.Command("sh", "/home/system/components/edgeboxctl/scripts/cloudflared_tunnel_delete.sh") + stdout, err := cmd.StdoutPipe() + if err != nil { + panic(err) + } + scanner := bufio.NewScanner(stdout) + err = cmd.Start() + if err != nil { + panic(err) + } + for scanner.Scan() { + fmt.Println(scanner.Text()) + text := scanner.Text() + fmt.Println(text) + } + if scanner.Err() != nil { + cmd.Process.Kill() + cmd.Wait() + panic(scanner.Err()) + } +} + +// InstallTunnelService: Installs the tunnel service +func InstallTunnelService(config string) { + fmt.Println("Installing cloudflared service.") + cmd := exec.Command("cloudflared", "--config", config, "service", "install") + cmd.Start() + cmd.Wait() +} + +// RemoveTunnelService: Removes the tunnel service +func RemoveTunnelService() { + wsPath := utils.GetPath(utils.WsPath) + fmt.Println("Removing possibly previous service install.") + cmd := exec.Command("cloudflared", "service", "uninstall") + cmd.Start() + cmd.Wait() + + fmt.Println("Removing cloudflared files") + cmdargs := []string{"-rf", "/home/system/.cloudflared"} + utils.Exec(wsPath, "rm", cmdargs) + cmdargs = []string{"-rf", "/etc/cloudflared/config.yml"} + utils.Exec(wsPath, "rm", cmdargs) + cmdargs = []string{"-rf", "/root/.cloudflared/cert.pem"} + utils.Exec(wsPath, "rm", cmdargs) +} + diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 779017e..dcf3dd9 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -7,12 +7,17 @@ import ( "log" "strconv" "time" + "os/exec" + "strings" + "os" + "bufio" "github.com/edgebox-iot/edgeboxctl/internal/diagnostics" "github.com/edgebox-iot/edgeboxctl/internal/edgeapps" "github.com/edgebox-iot/edgeboxctl/internal/storage" "github.com/edgebox-iot/edgeboxctl/internal/system" "github.com/edgebox-iot/edgeboxctl/internal/utils" + _ "github.com/go-sql-driver/mysql" // Mysql Driver _ "github.com/mattn/go-sqlite3" // SQlite Driver ) @@ -29,10 +34,7 @@ type Task struct { } type taskSetupTunnelArgs struct { - BootnodeAddress string `json:"bootnode_address"` - BootnodeToken string `json:"bootnode_token"` - AssignedAddress string `json:"assigned_address"` - NodeName string `json:"node_name"` + DomainName string `json:"domain_name"` } type taskStartEdgeAppArgs struct { @@ -64,6 +66,7 @@ type taskEnablePublicDashboardArgs struct { InternetURL string `json:"internet_url"` } + const STATUS_CREATED int = 0 const STATUS_EXECUTING int = 1 const STATUS_FINISHED int = 2 @@ -134,16 +137,36 @@ func ExecuteTask(task Task) Task { switch task.Task { case "setup_tunnel": - log.Println("Setting up bootnode connection...") + log.Println("Setting up Cloudflare Tunnel...") var args taskSetupTunnelArgs err := json.Unmarshal([]byte(task.Args.String), &args) if err != nil { - log.Printf("Error reading arguments of setup_bootnode task: %s", err) + log.Printf("Error reading arguments of setup_tunnel task: %s", err) + status := "{\"status\": \"error\", \"message\": \"The Domain Name you are going to Authorize must be provided beforehand! Please insert a domain name and try again.\"}" + utils.WriteOption("TUNNEL_STATUS", status) } else { taskResult := taskSetupTunnel(args) task.Result = sql.NullString{String: taskResult, Valid: true} } + case "start_tunnel": + + log.Println("Starting Cloudflare Tunnel...") + taskResult := taskStartTunnel() + task.Result = sql.NullString{String: taskResult, Valid: true} + + case "stop_tunnel": + + log.Println("Stopping Cloudflare Tunnel...") + taskResult := taskStopTunnel() + task.Result = sql.NullString{String: taskResult, Valid: true} + + case "disable_tunnel": + + log.Println("Disabling Cloudflare Tunnel...") + taskResult := taskDisableTunnel() + task.Result = sql.NullString{String: taskResult, Valid: true} + case "install_edgeapp": log.Println("Installing EdgeApp...") @@ -328,19 +351,141 @@ func ExecuteSchedules(tick int) { func taskSetupTunnel(args taskSetupTunnelArgs) string { fmt.Println("Executing taskSetupTunnel") + wsPath := utils.GetPath(utils.WsPath) - wsPath := utils.GetPath(utils.WsPath) - cmdargs := []string{"gen", "--name", args.NodeName, "--token", args.BootnodeToken, args.BootnodeAddress + ":8655", "--prefix", args.AssignedAddress} - utils.Exec(wsPath, "tinc-boot", cmdargs) + // Stop a the service if it is running + system.StopService("cloudflared") - cmdargs = []string{"start", "tinc@dnet"} - utils.Exec(wsPath, "systemctl", cmdargs) + // Uninstall the service if it is installed + system.RemoveTunnelService() - cmdargs = []string{"enable", "tinc@dnet"} - utils.Exec(wsPath, "systemctl", cmdargs) + fmt.Println("Creating cloudflared folder") + cmdargs := []string{"/home/system/.cloudflared"} + utils.Exec(wsPath, "mkdir", cmdargs) - output := "OK" // Better check / logging of command execution result. - return output + cmd := exec.Command("sh", "/home/system/components/edgeboxctl/scripts/cloudflared_login.sh") + stdout, err := cmd.StdoutPipe() + if err != nil { + panic(err) + } + scanner := bufio.NewScanner(stdout) + err = cmd.Start() + if err != nil { + panic(err) + } + url := "" + for scanner.Scan() { + fmt.Println(scanner.Text()) + text := scanner.Text() + if strings.Contains(text, "https://") { + url = text + fmt.Println("Tunnel setup is requesting auth with URL: " + url) + status := "{\"status\": \"waiting\", \"login_link\": \"" + url + "\"}" + utils.WriteOption("TUNNEL_STATUS", status) + break + } + } + if scanner.Err() != nil { + cmd.Process.Kill() + cmd.Wait() + panic(scanner.Err()) + } + + go func() { + fmt.Println("Running async") + cmd.Wait() + + // Keep retrying to read cert.pem file until it is created + // When running as a service, the cert is saved to a different folder, + // so we check both :) + for { + _, err := os.Stat("/home/system/.cloudflared/cert.pem") + _, err2 := os.Stat("/root/.cloudflared/cert.pem") + if err == nil || err2 == nil { + fmt.Println("cert.pem file detected") + break + } + time.Sleep(1 * time.Second) + fmt.Println("Waiting for cert.pem file to be created") + } + + fmt.Println("Tunnel auth setup finished without errors.") + status := "{\"status\": \"starting\", \"login_link\": \"" + url + "\"}" + utils.WriteOption("TUNNEL_STATUS", status) + + // Remove old tunnel if it exists, and create from scratch + system.DeleteTunnel() + + // Create new tunnel (destination config file is param) + system.CreateTunnel("/home/system/.cloudflared/config.yml") + + fmt.Println("Creating DNS Routes for @ and *.") + cmd = exec.Command("cloudflared", "tunnel", "route", "dns", "-f" ,"edgebox", "*." + args.DomainName) + cmd.Start() + err = cmd.Wait() + if err != nil { + log.Fatal(err) + } + + cmd = exec.Command("cloudflared", "tunnel", "route", "dns", "-f" ,"edgebox", args.DomainName) + cmd.Start() + err = cmd.Wait() + if err != nil { + log.Fatal(err) + } + + domainNameInfo := args.DomainName + utils.WriteOption("DOMAIN_NAME", domainNameInfo) + + // Install service with given config file + system.InstallTunnelService("/home/system/.cloudflared/config.yml") + + // Start the service + system.StartService("cloudflared") + + if err != nil { + fmt.Println("Tunnel auth setup finished with errors.") + status := "{\"status\": \"error\", \"login_link\": \"" + url + "\"}" + utils.WriteOption("TUNNEL_STATUS", status) + log.Fatal(err) + } else { + fmt.Println("Tunnel auth setup finished without errors.") + status := "{\"status\": \"connected\", \"login_link\": \"" + url + "\", \"domain\": \"" + args.DomainName + "\"}" + utils.WriteOption("TUNNEL_STATUS", status) + } + + fmt.Println("Finished running async") + }() + + return "{\"url\": \"" + url + "\"}" +} + +func taskStartTunnel() string { + fmt.Println("Executing taskStartTunnel") + system.StartService("cloudflared") + domainName := utils.ReadOption("DOMAIN_NAME") + status := "{\"status\": \"connected\", \"domain\": \"" + domainName + "\"}" + utils.WriteOption("TUNNEL_STATUS", status) + return "{\"status\": \"ok\"}" +} + +func taskStopTunnel() string { + fmt.Println("Executing taskStopTunnel") + system.StopService("cloudflared") + domainName := utils.ReadOption("DOMAIN_NAME") + status := "{\"status\": \"stopped\", \"domain\": \"" + domainName + "\"}" + utils.WriteOption("TUNNEL_STATUS", status) + return "{\"status\": \"ok\"}" +} + +func taskDisableTunnel() string { + fmt.Println("Executing taskDisableTunnel") + system.StopService("cloudflared") + system.DeleteTunnel() + system.RemoveTunnelService() + utils.DeleteOption("DOMAIN_NAME") + utils.DeleteOption("TUNNEL_STATUS") + return "{\"status\": \"ok\"}" } func taskInstallEdgeApp(args taskInstallEdgeAppArgs) string { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index e35ac41..e56b7a7 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -192,3 +192,47 @@ func WriteOption(optionKey string, optionValue string) { db.Close() } + +// ReadOption : Reads a key value pair option from the api shared database +func ReadOption(optionKey string) string { + + db, err := sql.Open("sqlite3", GetSQLiteDbConnectionDetails()) + + if err != nil { + log.Fatal(err.Error()) + } + + var optionValue string + + err = db.QueryRow("SELECT value FROM option WHERE name = ?", optionKey).Scan(&optionValue) + + if err != nil { + log.Fatal(err.Error()) + } + + db.Close() + + return optionValue +} + +// DeleteOption : Deletes a key value pair option from the api shared database +func DeleteOption(optionKey string) { + + db, err := sql.Open("sqlite3", GetSQLiteDbConnectionDetails()) + + if err != nil { + log.Fatal(err.Error()) + } + + statement, err := db.Prepare("DELETE FROM option WHERE name = ?;") // Prepare SQL Statement + if err != nil { + log.Fatal(err.Error()) + } + + _, err = statement.Exec(optionKey) // Execute SQL Statement + if err != nil { + log.Fatal(err.Error()) + } + + db.Close() +} diff --git a/scripts/cloudflared_login.sh b/scripts/cloudflared_login.sh new file mode 100644 index 0000000..12e985e --- /dev/null +++ b/scripts/cloudflared_login.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +echo "Starting script login" +cloudflared tunnel login 2>&1 | tee /home/system/components/edgeboxctl/scripts/output.log & +echo "sleeping 5 seconds" +sleep 5 \ No newline at end of file diff --git a/scripts/cloudflared_tunnel_create.sh b/scripts/cloudflared_tunnel_create.sh new file mode 100755 index 0000000..097d6b0 --- /dev/null +++ b/scripts/cloudflared_tunnel_create.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +echo "Starting script create" +# script -q -c "cloudflared tunnel login 2>&1 | tee /app/output.log" & +cloudflared tunnel create edgebox 2>&1 | tee /home/system/components/edgeboxctl/scripts/output.log +echo "sleeping 5 seconds" +sleep 5 \ No newline at end of file diff --git a/scripts/cloudflared_tunnel_delete.sh b/scripts/cloudflared_tunnel_delete.sh new file mode 100755 index 0000000..5a237bb --- /dev/null +++ b/scripts/cloudflared_tunnel_delete.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +echo "Starting script delete" +TUNNEL_ORIGIN_CERT=/home/system/.cloudflared/cert.pem +cloudflared tunnel delete edgebox 2>&1 | tee /home/system/components/edgeboxctl/scripts/output.log & +echo "sleeping 5 seconds" +sleep 5 \ No newline at end of file From f7823b9e20ef01213c52f62f85c033d73f9c02e3 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Mon, 17 Apr 2023 20:28:06 +0200 Subject: [PATCH 02/20] Added make build- commands --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index c09a58b..8b0bc74 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,15 @@ build-prod: build-cloud: GOOS=linux GOARCH=amd64 RELEASE=cloud make build +build-arm64: + GOOS=linux GOARCH=arm64 RELEASE=prod make build + +build-armhf: + GOOS=linux GOARCH=arm RELEASE=prod make build + +build-amd64: + GOOS=linux GOARCH=amd64 RELEASE=prod make build + build: @echo "Building ${GOOS}-${GOARCH}" GOOS=${GOOS} GOARCH=${GOARCH} go build \ From a66ff40e6545e1779620087dc03c0857514b4abb Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Mon, 29 May 2023 21:52:27 +0200 Subject: [PATCH 03/20] Backups feature --- Makefile | 5 +- internal/backups/backups.go | 28 ++++ internal/system/system.go | 16 +++ internal/tasks/tasks.go | 254 ++++++++++++++++++++++++++++++++++++ internal/utils/utils.go | 19 ++- 5 files changed, 315 insertions(+), 7 deletions(-) create mode 100644 internal/backups/backups.go diff --git a/Makefile b/Makefile index 8b0bc74..fd3414c 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,6 @@ build-arm64: build-armhf: GOOS=linux GOARCH=arm RELEASE=prod make build -build-amd64: - GOOS=linux GOARCH=amd64 RELEASE=prod make build - build: @echo "Building ${GOOS}-${GOARCH}" GOOS=${GOOS} GOARCH=${GOARCH} go build \ @@ -52,7 +49,7 @@ install-cloud: build-cloud @echo "To start edgeboxctl run: systemctl start edgeboxctl" install-prod: build-prod - -sudo systemctl stop edgeboxctl + sudo systemctl stop edgeboxctl sudo rm -rf /usr/local/bin/edgeboxctl /lib/systemd/system/edgeboxctl.service sudo cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl sudo cp ./edgeboxctl.service /lib/systemd/system/edgeboxctl.service diff --git a/internal/backups/backups.go b/internal/backups/backups.go new file mode 100644 index 0000000..68cb2dc --- /dev/null +++ b/internal/backups/backups.go @@ -0,0 +1,28 @@ +package backups + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/edgebox-iot/edgeboxctl/internal/diagnostics" + "github.com/edgebox-iot/edgeboxctl/internal/utils" + "github.com/shirou/gopsutil/disk" +) + +// Repository : Struct representing the backup repository of a device in the system +type Repository struct { + ID string `json:"id"` + FileCount int64 `json:"file_count"` + Size string `json:"size"` + Snapshots []Snapshot `json:"snapshots"` + Status string `json:"status"` + UsageStat UsageStat `json:"usage_stat"` +} + +// Snapshot : Struct representing a single snapshot in the backup repository +type Snapshot struct { + ID string `json:"id"` + time string `json:"time"` +} \ No newline at end of file diff --git a/internal/system/system.go b/internal/system/system.go index 95a6538..a8a7791 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -130,6 +130,22 @@ func GetServiceStatus(serviceID string) string { return utils.Exec(wsPath, "systemctl", cmdargs) } +func CreateBackupsPasswordFile(password string) { + // Create a password file for backups + backupPasswordFile := utils.GetPath(utils.BackupPasswordFileLocation) + backupPasswordFileDir := filepath.Dir(backupPasswordFile) + + if _, err := os.Stat(backupPasswordFileDir); os.IsNotExist(err) { + os.MkdirAll(backupPasswordFileDir, 0755) + } + + // Write the password to the file, overriting an existing file + err := ioutil.WriteFile(backupPasswordFile, []byte(password), 0644) + if err != nil { + panic(err) + } +} + // CreateTunnel: Creates a tunnel via cloudflared, needs to be authenticated first func CreateTunnel(configDestination string) { fmt.Println("Creating Tunnel for Edgebox.") diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index dcf3dd9..84e0e2c 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -66,6 +66,14 @@ type taskEnablePublicDashboardArgs struct { InternetURL string `json:"internet_url"` } +type taskSetupBackupsArgs struct { + Service string `json:"service"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + RepositoryName string `json:"repository_name"` + RepositoryPassword string `json:"repository_password"` +} + const STATUS_CREATED int = 0 const STATUS_EXECUTING int = 1 @@ -124,6 +132,7 @@ func ExecuteTask(task Task) Task { formatedDatetime := utils.GetSQLiteFormattedDateTime(time.Now()) + fmt.Println("Changing task status to executing: " + task.Task) _, err = statement.Exec(STATUS_EXECUTING, formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement if err != nil { log.Fatal(err.Error()) @@ -135,6 +144,34 @@ func ExecuteTask(task Task) Task { log.Println("Task: " + task.Task) log.Println("Args: " + task.Args.String) switch task.Task { + case "setup_backups": + + log.Println("Setting up Backups Destination...") + var args taskSetupBackupsArgs + err := json.Unmarshal([]byte(task.Args.String), &args) + if err != nil { + log.Println("Error reading arguments of setup_backups task: %s", err) + } else { + taskResult := taskSetupBackups(args) + taskResultBool := true + // Check if returned taskResult string contains "error" + if strings.Contains(taskResult, "error") { + taskResultBool = false + } + task.Result = sql.NullString{String: taskResult, Valid: taskResultBool} + } + + case "start_backup": + + log.Println("Backing up Edgebox...") + taskResult := taskBackup() + taskResultBool := true + // Check if returned taskResult string contains "error" + if strings.Contains(taskResult, "error") { + taskResultBool = false + } + task.Result = sql.NullString{String: taskResult, Valid: taskResultBool} + case "setup_tunnel": log.Println("Setting up Cloudflare Tunnel...") @@ -272,11 +309,13 @@ func ExecuteTask(task Task) Task { formatedDatetime = utils.GetSQLiteFormattedDateTime(time.Now()) if task.Result.Valid { + fmt.Println("Task Result: " + task.Result.String) _, err = statement.Exec(STATUS_FINISHED, task.Result.String, formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement with result info if err != nil { log.Fatal(err.Error()) } } else { + fmt.Println("Error executing task with result: " + task.Result.String) _, err = statement.Exec(STATUS_ERROR, "Error", formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement with Error info if err != nil { log.Fatal(err.Error()) @@ -338,6 +377,8 @@ func ExecuteSchedules(tick int) { if tick%30 == 0 { // Executing every 30 ticks log.Println(taskGetEdgeApps()) + // RESET SOME VARIABLES HERE IF NEEDED, SINCE SYSTEM IS UNBLOCKED + utils.WriteOption("BACKUP_IS_WORKING", "0") } if tick%60 == 0 { @@ -345,10 +386,223 @@ func ExecuteSchedules(tick int) { log.Println("System IP is: " + ip) } + if tick%3600 == 0 { + // Executing every 3600 ticks (1 hour) + backup := taskAutoBackup() + log.Println("Auto Backup: " + backup) + } + // Just add a schedule here if you need a custom one (every "tick hour", every "tick day", etc...) } +func taskSetupBackups(args taskSetupBackupsArgs) string { + fmt.Println("Executing taskSetupBackups" + args.Service) + // ... + service_url := "" + key_id_name := "AWS_ACCESS_KEY_ID" + key_secret_name := "AWS_SECRET_ACCESS_KEY" + repo_location := "/home/system/components/apps/" + service_found := false + + switch args.Service { + case "s3": + service_url = "s3.amazonaws.com/" + service_found = true + case "b2": + service_url = "" + key_id_name = "B2_ACCOUNT_ID" + key_secret_name = "B2_ACCOUNT_KEY" + service_found = true + case "wasabi": + service_found = true + service_url = "s3.wasabisys.com/" + } + + if !service_found { + fmt.Println("Service not found") + return "{\"status\": \"error\", \"message\": \"Service not found\"}" + } + + fmt.Println("Creating env vars for authentication with backup service") + os.Setenv(key_id_name, args.AccessKeyID) + os.Setenv(key_secret_name, args.SecretAccessKey) + + fmt.Println("Creating restic password file") + + system.CreateBackupsPasswordFile(args.RepositoryPassword) + + fmt.Println("Initializing restic repository") + utils.WriteOption("BACKUP_IS_WORKING", "1") + + cmdArgs := []string{"-r", args.Service + ":" + service_url + args.RepositoryName + ":" + repo_location, "init", "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"} + + result := utils.ExecAndStream(repo_location, "restic", cmdArgs) + + utils.WriteOption("BACKUP_IS_WORKING", "0") + + // See if result contains the substring "Fatal:" + if strings.Contains(result, "Fatal:") { + fmt.Println("Error initializing restic repository") + + utils.WriteOption("BACKUP_STATUS", "error") + utils.WriteOption("BACKUP_ERROR_MESSAGE", result) + + return "{\"status\": \"error\", \"message\": \"" + result + "\"}" + } + + // Save options to database + utils.WriteOption("BACKUP_STATUS", "initiated") + utils.WriteOption("BACKUP_SERVICE", args.Service) + utils.WriteOption("BACKUP_SERVICE_URL", service_url) + utils.WriteOption("BACKUP_REPOSITORY_NAME", args.RepositoryName) + utils.WriteOption("BACKUP_REPOSITORY_PASSWORD", args.RepositoryPassword) + utils.WriteOption("BACKUP_REPOSITORY_ACCESS_KEY_ID", args.AccessKeyID) + utils.WriteOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY", args.SecretAccessKey) + utils.WriteOption("BACKUP_REPOSITORY_LOCATION", repo_location) + + // Populate Stats right away + taskGetBackupStatus() + + return "{\"status\": \"ok\"}" + +} + +func taskRemoveBackups() string { + + fmt.Println("Executing taskRemoveBackups") + + // ... This deletes the restic repository + // cmdArgs := []string{"-r", "s3:https://s3.amazonaws.com/edgebox-backups:/home/system/components/apps/", "forget", "latest", "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"} + + utils.WriteOption("BACKUP_STATUS", "") + utils.WriteOption("BACKUP_IS_WORKING", "0") + + return "{\"status\": \"ok\"}" + +} + +func taskBackup() string { + fmt.Println("Executing taskBackup") + + // Load Backup Options + backup_service := utils.ReadOption("BACKUP_SERVICE") + backup_service_url := utils.ReadOption("BACKUP_SERVICE_URL") + backup_repository_name := utils.ReadOption("BACKUP_REPOSITORY_NAME") + // backup_repository_password := utils.ReadOption("BACKUP_REPOSITORY_PASSWORD") + backup_repository_access_key_id := utils.ReadOption("BACKUP_REPOSITORY_ACCESS_KEY_ID") + backup_repository_secret_access_key := utils.ReadOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY") + backup_repository_location := utils.ReadOption("BACKUP_REPOSITORY_LOCATION") + + key_id_name := "AWS_ACCESS_KEY_ID" + key_secret_name := "AWS_SECRET_ACCESS_KEY" + service_found := false + + switch backup_service { + case "s3": + service_found = true + case "b2": + key_id_name = "B2_ACCOUNT_ID" + key_secret_name = "B2_ACCOUNT_KEY" + service_found = true + case "wasabi": + service_found = true + } + + if !service_found { + fmt.Println("Service not found") + return "{\"status\": \"error\", \"message\": \"Backup Service not found\"}" + } + + fmt.Println("Creating env vars for authentication with backup service") + fmt.Println(key_id_name) + os.Setenv(key_id_name, backup_repository_access_key_id) + fmt.Println(key_secret_name) + os.Setenv(key_secret_name, backup_repository_secret_access_key) + + + utils.WriteOption("BACKUP_IS_WORKING", "1") + + // ... This backs up the restic repository + cmdArgs := []string{"-r", backup_service + ":" + backup_service_url + backup_repository_name + ":" + backup_repository_location, "backup", backup_repository_location, "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"} + result := utils.ExecAndStream(backup_repository_location, "restic", cmdArgs) + + utils.WriteOption("BACKUP_IS_WORKING", "0") + // Write as Unix timestamp + utils.WriteOption("BACKUP_LAST_RUN", strconv.FormatInt(time.Now().Unix(), 10)) + + // See if result contains the substring "Fatal:" + if strings.Contains(result, "Fatal:") { + fmt.Println("Error backing up") + utils.WriteOption("BACKUP_STATUS", "error") + utils.WriteOption("BACKUP_ERROR_MESSAGE", result) + return "{\"status\": \"error\", \"message\": \"" + result + "\"}" + } + + utils.WriteOption("BACKUP_STATUS", "working") + taskGetBackupStatus() + return "{\"status\": \"ok\"}" + +} + +func taskAutoBackup() string { + fmt.Println("Executing taskAutoBackup") + + // Get Backup Status + backup_status := utils.ReadOption("BACKUP_STATUS") + // We only backup is the status is "working" + if backup_status == "working" { + return taskBackup() + } else { + fmt.Println("Backup status is not working... skipping") + return "{\"status\": \"skipped\"}" + } +} + +func taskGetBackupStatus() string { + fmt.Println("Executing taskGetBackupStatus") + + // Load Backup Options + backup_service := utils.ReadOption("BACKUP_SERVICE") + backup_service_url := utils.ReadOption("BACKUP_SERVICE_URL") + backup_repository_name := utils.ReadOption("BACKUP_REPOSITORY_NAME") + // backup_repository_password := utils.ReadOption("BACKUP_REPOSITORY_PASSWORD") + backup_repository_access_key_id := utils.ReadOption("BACKUP_REPOSITORY_ACCESS_KEY_ID") + backup_repository_secret_access_key := utils.ReadOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY") + backup_repository_location := utils.ReadOption("BACKUP_REPOSITORY_LOCATION") + + key_id_name := "AWS_ACCESS_KEY_ID" + key_secret_name := "AWS_SECRET_ACCESS_KEY" + service_found := false + + switch backup_service { + case "s3": + service_found = true + case "b2": + key_id_name = "B2_ACCOUNT_ID" + key_secret_name = "B2_ACCOUNT_KEY" + service_found = true + case "wasabi": + service_found = true + } + + if !service_found { + fmt.Println("Service not found") + return "{\"status\": \"error\", \"message\": \"Backup Service not found\"}" + } + + fmt.Println("Creating env vars for authentication with backup service") + os.Setenv(key_id_name, backup_repository_access_key_id) + os.Setenv(key_secret_name, backup_repository_secret_access_key) + + // ... This gets the restic repository status + cmdArgs := []string{"-r", backup_service + ":" + backup_service_url + backup_repository_name + ":" + backup_repository_location, "stats", "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"} + utils.WriteOption("BACKUP_STATS", utils.ExecAndStream(backup_repository_location, "restic", cmdArgs)) + + return "{\"status\": \"ok\"}" + +} + func taskSetupTunnel(args taskSetupTunnelArgs) string { fmt.Println("Executing taskSetupTunnel") wsPath := utils.GetPath(utils.WsPath) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index e56b7a7..9364bcb 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -16,7 +16,7 @@ import ( ) // ExecAndStream : Runs a terminal command, but streams progress instead of outputting. Ideal for long lived process that need to be logged. -func ExecAndStream(path string, command string, args []string) { +func ExecAndStream(path string, command string, args []string) string { cmd := exec.Command(command, args...) @@ -27,13 +27,17 @@ func ExecAndStream(path string, command string, args []string) { err := cmd.Run() + outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes()) + + returnVal := outStr if err != nil { fmt.Printf("cmd.Run() failed with %s\n", err) + returnVal = errStr } - outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes()) fmt.Printf("\nout:\n%s\nerr:\n%s\n", outStr, errStr) + return returnVal } // Exec : Runs a terminal Command, catches and logs errors, returns the result. @@ -100,6 +104,7 @@ func GetSQLiteFormattedDateTime(t time.Time) string { return formatedDatetime } +const BackupPasswordFileLocation string = "backupPasswordFileLocation" const CloudEnvFileLocation string = "cloudEnvFileLocation" const ApiEnvFileLocation string = "apiEnvFileLocation" const ApiPath string = "apiPath" @@ -159,6 +164,14 @@ func GetPath(pathKey string) string { targetPath = "/home/system/components/ws/" } + case BackupPasswordFileLocation: + + if env["BACKUP_PASSWORD_FILE_LOCATION"] != "" { + targetPath = env["BACKUP_PASSWORD_FILE_LOCATION"] + } else { + targetPath = "/home/system/components/backups/pw.txt" + } + default: log.Printf("path_key %s nonexistant in GetPath().\n", pathKey) @@ -207,7 +220,7 @@ func ReadOption(optionKey string) string { err = db.QueryRow("SELECT value FROM option WHERE name = ?", optionKey).Scan(&optionValue) if err != nil { - log.Fatal(err.Error()) + log.Println(err.Error()) } db.Close() From 04cdba7479b5183f5eed3d7fb4ae0185bff8e56c Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 30 May 2023 20:38:06 +0200 Subject: [PATCH 04/20] Run next backup based on time elapsed since last backup record --- internal/tasks/tasks.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 84e0e2c..a49a058 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -379,6 +379,29 @@ func ExecuteSchedules(tick int) { log.Println(taskGetEdgeApps()) // RESET SOME VARIABLES HERE IF NEEDED, SINCE SYSTEM IS UNBLOCKED utils.WriteOption("BACKUP_IS_WORKING", "0") + + // Check is Last Backup time (in unix time) is older than 1 h + lastBackup := utils.ReadOption("BACKUP_LAST_RUN") + if lastBackup != "" { + lastBackupTime, err := strconv.ParseInt(lastBackup, 10, 64) + if err != nil { + log.Println("Error parsing last backup time: " + err.Error()) + } else { + secondsSinceLastBackup := time.Now().Unix() - lastBackupTime + if secondsSinceLastBackup > 3600 { + // If last backup is older than 1 hour, set BACKUP_IS_WORKING to 0 + log.Println("Last backup was older than 1 hour, performing auto backup...") + log.Println(taskAutoBackup()) + } else { + + log.Println("Last backup is " + string(secondsSinceLastBackup) + " seconds old (less than 1 hour ago), skipping auto backup...") + + } + } + } else { + log.Println("Last backup time not found, skipping performing auto backup...") + } + } if tick%60 == 0 { @@ -388,8 +411,6 @@ func ExecuteSchedules(tick int) { if tick%3600 == 0 { // Executing every 3600 ticks (1 hour) - backup := taskAutoBackup() - log.Println("Auto Backup: " + backup) } // Just add a schedule here if you need a custom one (every "tick hour", every "tick day", etc...) From 13275f80a56e7bf465aef948389b5021d1cee567 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 30 May 2023 21:20:30 +0200 Subject: [PATCH 05/20] Temporarily commented imports on backups module --- internal/backups/backups.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/backups/backups.go b/internal/backups/backups.go index 68cb2dc..d89ed3b 100644 --- a/internal/backups/backups.go +++ b/internal/backups/backups.go @@ -1,15 +1,15 @@ package backups -import ( - "fmt" - "os" - "path/filepath" - "strings" +// import ( +// "fmt" +// "os" +// "path/filepath" +// "strings" - "github.com/edgebox-iot/edgeboxctl/internal/diagnostics" - "github.com/edgebox-iot/edgeboxctl/internal/utils" - "github.com/shirou/gopsutil/disk" -) +// "github.com/edgebox-iot/edgeboxctl/internal/diagnostics" +// "github.com/edgebox-iot/edgeboxctl/internal/utils" +// "github.com/shirou/gopsutil/disk" +// ) // Repository : Struct representing the backup repository of a device in the system type Repository struct { From 335b7ec29ee00bbafdabb06bb815f2f12428ddf3 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 30 May 2023 21:21:24 +0200 Subject: [PATCH 06/20] Commented UsageStat member of Repository --- internal/backups/backups.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backups/backups.go b/internal/backups/backups.go index d89ed3b..8b3940c 100644 --- a/internal/backups/backups.go +++ b/internal/backups/backups.go @@ -18,7 +18,7 @@ type Repository struct { Size string `json:"size"` Snapshots []Snapshot `json:"snapshots"` Status string `json:"status"` - UsageStat UsageStat `json:"usage_stat"` + // UsageStat UsageStat `json:"usage_stat"` } // Snapshot : Struct representing a single snapshot in the backup repository From df13bf705d3182c130804da2c589ed5805ac06fe Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 30 May 2023 21:22:31 +0200 Subject: [PATCH 07/20] Removed formatting directive from Println call --- internal/tasks/tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index a49a058..2b2f501 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -150,7 +150,7 @@ func ExecuteTask(task Task) Task { var args taskSetupBackupsArgs err := json.Unmarshal([]byte(task.Args.String), &args) if err != nil { - log.Println("Error reading arguments of setup_backups task: %s", err) + log.Println("Error reading arguments of setup_backups task: " + err) } else { taskResult := taskSetupBackups(args) taskResultBool := true From 113785def414813579ffa59319d9abc81afcf899 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 30 May 2023 21:23:33 +0200 Subject: [PATCH 08/20] Using Sprintf to convert from int64 to string of digits --- internal/tasks/tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 2b2f501..6585631 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -394,7 +394,7 @@ func ExecuteSchedules(tick int) { log.Println(taskAutoBackup()) } else { - log.Println("Last backup is " + string(secondsSinceLastBackup) + " seconds old (less than 1 hour ago), skipping auto backup...") + log.Println("Last backup is " + fmt.Sprint(secondsSinceLastBackup) + " seconds old (less than 1 hour ago), skipping auto backup...") } } From 8d0de36cdde4b7b2b3d8c17c5f0a62e375e9f459 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 30 May 2023 21:25:40 +0200 Subject: [PATCH 09/20] Removed err from log --- internal/tasks/tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 6585631..bd8b803 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -150,7 +150,7 @@ func ExecuteTask(task Task) Task { var args taskSetupBackupsArgs err := json.Unmarshal([]byte(task.Args.String), &args) if err != nil { - log.Println("Error reading arguments of setup_backups task: " + err) + log.Println("Error reading arguments of setup_backups task.") } else { taskResult := taskSetupBackups(args) taskResultBool := true From 284acb18a443a9ca342e8e1b280aaf1691a6275e Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Wed, 31 May 2023 00:03:23 +0000 Subject: [PATCH 10/20] Added restore functionality --- internal/edgeapps/edgeapps.go | 30 +++++++++ internal/tasks/tasks.go | 111 +++++++++++++++++++++++++++++++--- 2 files changed, 134 insertions(+), 7 deletions(-) diff --git a/internal/edgeapps/edgeapps.go b/internal/edgeapps/edgeapps.go index ab4cf8c..b285e70 100644 --- a/internal/edgeapps/edgeapps.go +++ b/internal/edgeapps/edgeapps.go @@ -296,6 +296,36 @@ func StopEdgeApp(ID string) EdgeAppStatus { } +// StopAllEdgeApps: Stops all EdgeApps and returns a count of how many were stopped +func StopAllEdgeApps() int { + edgeApps := GetEdgeApps() + appCount := 0 + for _, edgeApp := range edgeApps { + StopEdgeApp(edgeApp.ID) + appCount++ + } + + return appCount + +} + +// StartAllEdgeApps: Starts all EdgeApps and returns a count of how many were started +func StartAllEdgeApps() int { + edgeApps := GetEdgeApps() + appCount := 0 + for _, edgeApp := range edgeApps { + RunEdgeApp(edgeApp.ID) + appCount++ + } + + return appCount + +} + +func RestartEdgeAppsService() { + buildFrameworkContainers() +} + // EnableOnline : Write environment file and rebuild the necessary containers. Rebuilds containers in project (in case of change only) func EnableOnline(ID string, InternetURL string) MaybeEdgeApp { diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index bd8b803..d2c49f9 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -172,6 +172,16 @@ func ExecuteTask(task Task) Task { } task.Result = sql.NullString{String: taskResult, Valid: taskResultBool} + case "restore_backup": + log.Println("Attempting to Restore Last Backup to Edgebox") + taskResult := taskRestoreBackup() + taskResultBool := true + // Check if returned taskResult string contains "error" + if strings.Contains(taskResult, "error") { + taskResultBool = false + } + task.Result = sql.NullString{String: taskResult, Valid: taskResultBool} + case "setup_tunnel": log.Println("Setting up Cloudflare Tunnel...") @@ -462,6 +472,15 @@ func taskSetupBackups(args taskSetupBackupsArgs) string { utils.WriteOption("BACKUP_IS_WORKING", "0") + // Write backup settings to table + utils.WriteOption("BACKUP_SERVICE", args.Service) + utils.WriteOption("BACKUP_SERVICE_URL", service_url) + utils.WriteOption("BACKUP_REPOSITORY_NAME", args.RepositoryName) + utils.WriteOption("BACKUP_REPOSITORY_PASSWORD", args.RepositoryPassword) + utils.WriteOption("BACKUP_REPOSITORY_ACCESS_KEY_ID", args.AccessKeyID) + utils.WriteOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY", args.SecretAccessKey) + utils.WriteOption("BACKUP_REPOSITORY_LOCATION", repo_location) + // See if result contains the substring "Fatal:" if strings.Contains(result, "Fatal:") { fmt.Println("Error initializing restic repository") @@ -474,13 +493,6 @@ func taskSetupBackups(args taskSetupBackupsArgs) string { // Save options to database utils.WriteOption("BACKUP_STATUS", "initiated") - utils.WriteOption("BACKUP_SERVICE", args.Service) - utils.WriteOption("BACKUP_SERVICE_URL", service_url) - utils.WriteOption("BACKUP_REPOSITORY_NAME", args.RepositoryName) - utils.WriteOption("BACKUP_REPOSITORY_PASSWORD", args.RepositoryPassword) - utils.WriteOption("BACKUP_REPOSITORY_ACCESS_KEY_ID", args.AccessKeyID) - utils.WriteOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY", args.SecretAccessKey) - utils.WriteOption("BACKUP_REPOSITORY_LOCATION", repo_location) // Populate Stats right away taskGetBackupStatus() @@ -566,6 +578,91 @@ func taskBackup() string { } +func taskRestoreBackup() string { + fmt.Println("Executing taskRestoreBackup") + + // Load Backup Options + backup_service := utils.ReadOption("BACKUP_SERVICE") + backup_service_url := utils.ReadOption("BACKUP_SERVICE_URL") + backup_repository_name := utils.ReadOption("BACKUP_REPOSITORY_NAME") + // backup_repository_password := utils.ReadOption("BACKUP_REPOSITORY_PASSWORD") + backup_repository_access_key_id := utils.ReadOption("BACKUP_REPOSITORY_ACCESS_KEY_ID") + backup_repository_secret_access_key := utils.ReadOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY") + backup_repository_location := utils.ReadOption("BACKUP_REPOSITORY_LOCATION") + + key_id_name := "AWS_ACCESS_KEY_ID" + key_secret_name := "AWS_SECRET_ACCESS_KEY" + service_found := false + + switch backup_service { + case "s3": + service_found = true + case "b2": + key_id_name = "B2_ACCOUNT_ID" + key_secret_name = "B2_ACCOUNT_KEY" + service_found = true + case "wasabi": + service_found = true + } + + if !service_found { + fmt.Println("Service not found") + return "{\"status\": \"error\", \"message\": \"Backup Service not found\"}" + } + + fmt.Println("Creating env vars for authentication with backup service") + fmt.Println(key_id_name) + os.Setenv(key_id_name, backup_repository_access_key_id) + fmt.Println(key_secret_name) + os.Setenv(key_secret_name, backup_repository_secret_access_key) + + + utils.WriteOption("BACKUP_IS_WORKING", "1") + + fmt.Println("Stopping All EdgeApps") + // Stop All EdgeApps + edgeapps.StopAllEdgeApps() + + // Copy all files in /home/system/components/apps/ to a backup folder + fmt.Println("Copying all files in /home/system/components/apps/ to a backup folder") + os.MkdirAll(utils.GetPath(utils.EdgeAppsBackupPath + "temp/"), 0777) + utils.CopyDir(utils.GetPath(utils.EdgeAppsPath), utils.GetPath(utils.EdgeAppsBackupPath + "temp/")) + + fmt.Println("Removing all files in /home/system/components/apps/") + os.RemoveAll(utils.GetPath(utils.EdgeAppsPath)) + + // Create directory /home/system/components/apps/ + fmt.Println("Creating directory /home/system/components/apps/") + os.MkdirAll(utils.GetPath(utils.EdgeAppsPath), 0777) + + // ... This restores up the restic repository + cmdArgs := []string{"-r", backup_service + ":" + backup_service_url + backup_repository_name + ":" + backup_repository_location, "restore", "latest", "--target", "/", "--path", backup_repository_location, "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"} + result := utils.ExecAndStream(backup_repository_location, "restic", cmdArgs) + + taskGetBackupStatus() + + edgeapps.RestartEdgeAppsService() + + utils.WriteOption("BACKUP_IS_WORKING", "0") + + // See if result contains the substring "Fatal:" + if strings.Contains(result, "Fatal:") { + // Copy all files from backup folder to /home/system/components/apps/ + os.MkdirAll(utils.GetPath(utils.EdgeAppsPath), 0777) + utils.CopyDir(utils.GetPath(utils.EdgeAppsBackupPath + "temp/"), utils.GetPath(utils.EdgeAppsPath)) + + fmt.Println("Error restoring backup: ") + utils.WriteOption("BACKUP_STATUS", "error") + utils.WriteOption("BACKUP_ERROR_MESSAGE", result) + return "{\"status\": \"error\", \"message\": \"" + result + "\"}" + } + + utils.WriteOption("BACKUP_STATUS", "working") + taskGetBackupStatus() + return "{\"status\": \"ok\"}" + +} + func taskAutoBackup() string { fmt.Println("Executing taskAutoBackup") From e009c1f55d7e16943dad37d6c94e5bcafd1e0e09 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Sun, 4 Jun 2023 23:19:34 +0200 Subject: [PATCH 11/20] NetworkURL based on hostname --- internal/edgeapps/edgeapps.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/edgeapps/edgeapps.go b/internal/edgeapps/edgeapps.go index b285e70..83b10ea 100644 --- a/internal/edgeapps/edgeapps.go +++ b/internal/edgeapps/edgeapps.go @@ -9,6 +9,7 @@ import ( "github.com/joho/godotenv" + "github.com/edgebox-iot/edgeboxctl/internal/system" "github.com/edgebox-iot/edgeboxctl/internal/utils" ) @@ -97,7 +98,7 @@ func GetEdgeApp(ID string) MaybeEdgeApp { Status: GetEdgeAppStatus(ID), Services: GetEdgeAppServices(ID), InternetAccessible: edgeAppInternetAccessible, - NetworkURL: ID + ".edgebox.local", + NetworkURL: ID + system.GetHostname() + ".local", InternetURL: edgeAppInternetURL, }, Valid: true, From 8bf26a334c342e9df1d8996947bef8e222a5f3a9 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sat, 10 Jun 2023 17:29:54 +0200 Subject: [PATCH 12/20] Added system.CopyDir and EdgeappsBackupPath conf --- Makefile | 27 ++++++++------ internal/system/system.go | 76 +++++++++++++++++++++++++++++++++++++++ internal/tasks/tasks.go | 4 +-- internal/utils/utils.go | 8 +++++ 4 files changed, 103 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index fd3414c..2faaf7d 100644 --- a/Makefile +++ b/Makefile @@ -40,19 +40,26 @@ test: test-with-coverage: go test -tags=unit -timeout=600s -v ./... -coverprofile=coverage.out -install-cloud: build-cloud - systemctl stop edgeboxctl - cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl - cp ./edgeboxctl/edgeboxctl.service /lib/systemd/system/edgeboxctl.service - systemctl daemon-reload - @echo "Edgeboxctl installed successfully" - @echo "To start edgeboxctl run: systemctl start edgeboxctl" - -install-prod: build-prod +install: sudo systemctl stop edgeboxctl sudo rm -rf /usr/local/bin/edgeboxctl /lib/systemd/system/edgeboxctl.service sudo cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl sudo cp ./edgeboxctl.service /lib/systemd/system/edgeboxctl.service sudo systemctl daemon-reload @echo "Edgeboxctl installed successfully" - @echo "To start edgeboxctl run: systemctl start edgeboxctl" \ No newline at end of file + @echo "To start edgeboxctl run: systemctl start edgeboxctl" + +install-prod: build-prod install +install-cloud: build-cloud install +install-arm64: build-arm64 install +install-armhf: build-armhf install + +start: + systemctl start edgeboxctl + +stop: + systemctl stop edgeboxctl + +log: start + journalctl -fu edgeboxctl + diff --git a/internal/system/system.go b/internal/system/system.go index a8a7791..12e8c5d 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -6,6 +6,7 @@ import ( "strings" "log" "os" + "io" "os/exec" "bufio" "path/filepath" @@ -302,3 +303,78 @@ func RemoveTunnelService() { utils.Exec(wsPath, "rm", cmdargs) } +func CopyDir(src string, dest string) error { + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + if !srcInfo.IsDir() { + return fmt.Errorf("%s is not a directory", src) + } + + err = os.MkdirAll(dest, srcInfo.Mode()) + if err != nil { + return err + } + + items, err := ioutil.ReadDir(src) + if err != nil { + return err + } + + for _, item := range items { + srcPath := filepath.Join(src, item.Name()) + destPath := filepath.Join(dest, item.Name()) + + if item.IsDir() { + err = CopyDir(srcPath, destPath) + if err != nil { + fmt.Printf("error copying directory %s to %s: %s\n", srcPath, destPath, err.Error()) + } + } else { + err = CopyFile(srcPath, destPath) + if err != nil { + fmt.Printf("error copying file %s to %s: %s\n", srcPath, destPath, err.Error()) + } + } + } + + return nil +} + +func CopyFile(src string, dest string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + destFile, err := os.Create(dest) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, srcFile) + if err != nil { + return err + } + + err = destFile.Sync() + if err != nil { + return err + } + + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + + err = os.Chmod(dest, srcInfo.Mode()) + if err != nil { + return err + } + + return nil +} + diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index d2c49f9..a88c101 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -626,7 +626,7 @@ func taskRestoreBackup() string { // Copy all files in /home/system/components/apps/ to a backup folder fmt.Println("Copying all files in /home/system/components/apps/ to a backup folder") os.MkdirAll(utils.GetPath(utils.EdgeAppsBackupPath + "temp/"), 0777) - utils.CopyDir(utils.GetPath(utils.EdgeAppsPath), utils.GetPath(utils.EdgeAppsBackupPath + "temp/")) + system.CopyDir(utils.GetPath(utils.EdgeAppsPath), utils.GetPath(utils.EdgeAppsBackupPath + "temp/")) fmt.Println("Removing all files in /home/system/components/apps/") os.RemoveAll(utils.GetPath(utils.EdgeAppsPath)) @@ -649,7 +649,7 @@ func taskRestoreBackup() string { if strings.Contains(result, "Fatal:") { // Copy all files from backup folder to /home/system/components/apps/ os.MkdirAll(utils.GetPath(utils.EdgeAppsPath), 0777) - utils.CopyDir(utils.GetPath(utils.EdgeAppsBackupPath + "temp/"), utils.GetPath(utils.EdgeAppsPath)) + system.CopyDir(utils.GetPath(utils.EdgeAppsBackupPath + "temp/"), utils.GetPath(utils.EdgeAppsPath)) fmt.Println("Error restoring backup: ") utils.WriteOption("BACKUP_STATUS", "error") diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 9364bcb..16f1d87 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -109,6 +109,7 @@ const CloudEnvFileLocation string = "cloudEnvFileLocation" const ApiEnvFileLocation string = "apiEnvFileLocation" const ApiPath string = "apiPath" const EdgeAppsPath string = "edgeAppsPath" +const EdgeAppsBackupPath string = "edgeAppsBackupPath" const WsPath string = "wsPath" // GetPath : Returns either the hardcoded path, or a overwritten value via .env file at project root. Register paths here for seamless working code between dev and prod environments ;) @@ -156,6 +157,13 @@ func GetPath(pathKey string) string { targetPath = "/home/system/components/apps/" } + case EdgeAppsBackupPath: + if env["EDGEAPPS_BACKUP_PATH"] != "" { + targetPath = env["EDGEAPPS_BACKUP_PATH"] + } else { + targetPath = "/home/system/components/backups/" + } + case WsPath: if env["WS_PATH"] != "" { From 445bc41d0efd39991ff2d5435e28bca245763190 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Sat, 28 Oct 2023 18:43:18 +0200 Subject: [PATCH 13/20] Fixes to install make command, fixes to local network app url definition, run ws build on start and every 24 hours --- Makefile | 3 ++- internal/edgeapps/edgeapps.go | 2 +- internal/system/system.go | 8 ++++++++ internal/tasks/tasks.go | 12 ++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2faaf7d..ca58644 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,9 @@ test-with-coverage: install: sudo systemctl stop edgeboxctl - sudo rm -rf /usr/local/bin/edgeboxctl /lib/systemd/system/edgeboxctl.service + sudo rm -rf /usr/local/bin/edgeboxctl /usr/local/sbin/edgeboctl /lib/systemd/system/edgeboxctl.service sudo cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl + sudo cp ./bin/edgeboxctl /usr/local/sbin/edgeboxctl sudo cp ./edgeboxctl.service /lib/systemd/system/edgeboxctl.service sudo systemctl daemon-reload @echo "Edgeboxctl installed successfully" diff --git a/internal/edgeapps/edgeapps.go b/internal/edgeapps/edgeapps.go index 83b10ea..e0b66d5 100644 --- a/internal/edgeapps/edgeapps.go +++ b/internal/edgeapps/edgeapps.go @@ -98,7 +98,7 @@ func GetEdgeApp(ID string) MaybeEdgeApp { Status: GetEdgeAppStatus(ID), Services: GetEdgeAppServices(ID), InternetAccessible: edgeAppInternetAccessible, - NetworkURL: ID + system.GetHostname() + ".local", + NetworkURL: ID + "." + system.GetHostname() + ".local", InternetURL: edgeAppInternetURL, }, Valid: true, diff --git a/internal/system/system.go b/internal/system/system.go index 12e8c5d..82cb34d 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -100,6 +100,14 @@ func SetupCloudOptions() { utils.Exec("/", "rm", []string{cloudEnvFileLocationPath}) } +// StartWs: Starts the webserver service for Edgeapps +func StartWs() { + wsPath := utils.GetPath(utils.WsPath) + fmt.Println("Starting WS") + cmdargs := []string{"-b"} + utils.Exec(wsPath, "./ws", cmdargs) +} + // StartService: Starts a service func StartService(serviceID string) { wsPath := utils.GetPath(utils.WsPath) diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index a88c101..051cdfb 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -374,6 +374,7 @@ func ExecuteSchedules(tick int) { log.Println("Uptime is " + uptime + " seconds (" + system.GetUptimeFormatted() + ")") log.Println(taskGetStorageDevices()) + taskStartWs() log.Println(taskGetEdgeApps()) } @@ -423,6 +424,12 @@ func ExecuteSchedules(tick int) { // Executing every 3600 ticks (1 hour) } + if tick%86400 == 0 { + // Executing every 86400 ticks (+/1 day) + // Ensuring we run a normal build, setting up avahi domain names fresh in the network + taskStartWs() + } + // Just add a schedule here if you need a custom one (every "tick hour", every "tick day", etc...) } @@ -1000,3 +1007,8 @@ func taskSetupCloudOptions() { fmt.Println("Executing taskSetupCloudOptions") system.SetupCloudOptions() } + +func taskStartWs() { + fmt.Println("Executing taskStartWs") + system.StartWs() +} From ca7f4af7fe954f7393f44a87e27002d4ff8607c5 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 29 Oct 2023 20:48:14 +0100 Subject: [PATCH 14/20] Added Edgeapps Options Feature --- internal/edgeapps/edgeapps.go | 116 +++++++++++++++++++++++++++++++++- internal/system/system.go | 23 +++++++ internal/tasks/tasks.go | 108 ++++++++++++++++++++++++++++++- internal/utils/utils.go | 9 +++ 4 files changed, 254 insertions(+), 2 deletions(-) diff --git a/internal/edgeapps/edgeapps.go b/internal/edgeapps/edgeapps.go index e0b66d5..67bce80 100644 --- a/internal/edgeapps/edgeapps.go +++ b/internal/edgeapps/edgeapps.go @@ -6,7 +6,7 @@ import ( "os" "strings" "time" - + "github.com/joho/godotenv" "github.com/edgebox-iot/edgeboxctl/internal/system" @@ -23,6 +23,8 @@ type EdgeApp struct { InternetAccessible bool `json:"internet_accessible"` NetworkURL string `json:"network_url"` InternetURL string `json:"internet_url"` + Options []EdgeAppOption `json:"options"` + NeedsConfig bool `json:"needs_config"` } // MaybeEdgeApp : Boolean flag for validation of edgeapp existance @@ -43,8 +45,20 @@ type EdgeAppService struct { IsRunning bool `json:"is_running"` } +type EdgeAppOption struct { + Key string `json:"key"` + Value string `json:"value"` + DefaultValue string `json:"default_value"` + Format string `json:"format"` + Description string `json:"description"` + IsSecret bool `json:"is_secret"` + IsInstallLocked bool `json:"is_install_locked"` +} + const configFilename = "/edgebox-compose.yml" const envFilename = "/edgebox.env" +const optionsTemplateFilename = "/edgeapp.template.env" +const optionsEnvFilename = "/edgeapp.env" const runnableFilename = "/.run" const myEdgeAppServiceEnvFilename = "/myedgeapp.env" const defaultContainerOperationSleepTime time.Duration = time.Second * 10 @@ -63,6 +77,7 @@ func GetEdgeApp(ID string) MaybeEdgeApp { edgeAppName := ID edgeAppDescription := "" + edgeAppOptions := []EdgeAppOption{} edgeAppEnv, err := godotenv.Read(utils.GetPath(utils.EdgeAppsPath) + ID + envFilename) @@ -77,6 +92,103 @@ func GetEdgeApp(ID string) MaybeEdgeApp { } } + needsConfig := false + hasFilledOptions := false + edgeAppOptionsTemplate, err := godotenv.Read(utils.GetPath(utils.EdgeAppsPath) + ID + optionsTemplateFilename) + if err != nil { + log.Println("Error loading options template file for edgeapp " + edgeAppName) + } else { + // Try to read the edgeAppOptionsEnv file + edgeAppOptionsEnv, err := godotenv.Read(utils.GetPath(utils.EdgeAppsPath) + ID + optionsEnvFilename) + if err != nil { + log.Println("Error loading options env file for edgeapp " + edgeAppName) + } else { + hasFilledOptions = true + } + + for key, value := range edgeAppOptionsTemplate { + + optionFilledValue := "" + if hasFilledOptions { + // Check if key exists in edgeAppOptionsEnv + optionFilledValue = edgeAppOptionsEnv[key] + } + + format := "" + defaultValue := "" + description := "" + installLocked := false + + // Parse value to separate by | and get the format, installLocked, description and default value + // Format is the first element + // InstallLocked is the second element + // Description is the third element + // Default value is the fourth element + + valueSlices := strings.Split(value, "|") + if len(valueSlices) > 0 { + format = valueSlices[0] + } + if len(valueSlices) > 1 { + installLocked = valueSlices[1] == "true" + } + if len(valueSlices) > 2 { + description = valueSlices[2] + } + if len(valueSlices) > 3 { + defaultValue = valueSlices[3] + } + + // // If value contains ">|", then get everything that is to the right of it as the description + // // and get everything between "<>" as the format + // if strings.Contains(value, ">|") { + // description = strings.Split(value, ">|")[1] + // // Check if description has default value. That would be everything that is to the right of the last "|" + // if strings.Contains(description, "|") { + // defaultValue = strings.Split(description, "|")[1] + // description = strings.Split(description, "|")[0] + // } + + // value = strings.Split(value, ">|")[0] + // // Remove the initial < from value + // value = strings.TrimPrefix(value, "<") + // } else { + // // Trim initial < and final > from value + // value = strings.TrimPrefix(value, "<") + // value = strings.TrimSuffix(value, ">") + // } + + isSecret := false + + // Check if the lowercased key string contains the letters "pass", "secret", "key" + lowercaseKey := strings.ToLower(key) + // check if lowercaseInput contains "pass", "key", or "secret", or "token" + if strings.Contains(lowercaseKey, "pass") || + strings.Contains(lowercaseKey, "key") || + strings.Contains(lowercaseKey, "secret") || + strings.Contains(lowercaseKey, "token") { + isSecret = true + } + + currentOption := EdgeAppOption{ + Key: key, + Value: optionFilledValue, + DefaultValue: defaultValue, + Description: description, + Format: format, + IsSecret: isSecret, + IsInstallLocked: installLocked, + } + edgeAppOptions = append(edgeAppOptions, currentOption) + + if optionFilledValue == "" { + needsConfig = true + } + + } + } + + edgeAppInternetAccessible := false edgeAppInternetURL := "" @@ -100,6 +212,8 @@ func GetEdgeApp(ID string) MaybeEdgeApp { InternetAccessible: edgeAppInternetAccessible, NetworkURL: ID + "." + system.GetHostname() + ".local", InternetURL: edgeAppInternetURL, + Options: edgeAppOptions, + NeedsConfig: needsConfig, }, Valid: true, } diff --git a/internal/system/system.go b/internal/system/system.go index 82cb34d..42fe285 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -100,6 +100,29 @@ func SetupCloudOptions() { utils.Exec("/", "rm", []string{cloudEnvFileLocationPath}) } +func StartSystemLogger() { + fmt.Println("Starting system logger") + loggerPath := utils.GetPath(utils.LoggerPath) + utils.Exec(loggerPath, "make", []string{"start"}) +} + +// UpdateSystemLoggerServices: Updates the services.txt file with the services that are currently running +func UpdateSystemLoggerServices(services []string) { + fmt.Println("Updating system logger services:") + fmt.Println(services) + loggerPath := utils.GetPath(utils.LoggerPath) + + utils.Exec(loggerPath, "bash", []string{"-c", "rm services.txt && touch services.txt"}) + + for _, service := range services { + fmt.Println("Adding " + service + " to services.txt") + utils.Exec(loggerPath, "bash", []string{"-c", "echo " + service + " >> services.txt"}) + } + + // Add empty line at the end of file (best practice) + utils.Exec(loggerPath, "bash", []string{"-c", "echo '' >> services.txt"}) +} + // StartWs: Starts the webserver service for Edgeapps func StartWs() { wsPath := utils.GetPath(utils.WsPath) diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 051cdfb..e98720f 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -33,6 +33,12 @@ type Task struct { Updated string `json:"updated"` } +// TaskOption: Struct for Task Options (kv pair) +type TaskOption struct { + Key string `json:"key"` + Value string `json:"value"` +} + type taskSetupTunnelArgs struct { DomainName string `json:"domain_name"` } @@ -53,6 +59,12 @@ type taskStopEdgeAppArgs struct { ID string `json:"id"` } +type taskSetEdgeAppOptionsArgs struct { + ID string `json:"id"` + // Options should be an array of "key":"value" pairs + Options []TaskOption `json:"options"` +} + type taskEnableOnlineArgs struct { ID string `json:"id"` InternetURL string `json:"internet_url"` @@ -262,6 +274,19 @@ func ExecuteTask(task Task) Task { task.Result = sql.NullString{String: taskResult, Valid: true} } + case "set_edgeapp_options": + + log.Println("Setting EdgeApp Options...") + var args taskSetEdgeAppOptionsArgs + // {"id":"podgrab","options":{"PODGRAB_PASSWORD":"fumarmata"}} + err := json.Unmarshal([]byte(task.Args.String), &args) + if err != nil { + log.Printf("Error reading arguments of set_edgeapp_options task: %s", err) + } else { + taskResult := taskSetEdgeAppOptions(args) + task.Result = sql.NullString{String: taskResult, Valid: true} + } + case "enable_online": log.Println("Enabling online access to EdgeApp...") @@ -376,7 +401,7 @@ func ExecuteSchedules(tick int) { log.Println(taskGetStorageDevices()) taskStartWs() log.Println(taskGetEdgeApps()) - + taskUpdateSystemLoggerServices() } if tick%5 == 0 { @@ -388,6 +413,7 @@ func ExecuteSchedules(tick int) { if tick%30 == 0 { // Executing every 30 ticks log.Println(taskGetEdgeApps()) + taskUpdateSystemLoggerServices() // RESET SOME VARIABLES HERE IF NEEDED, SINCE SYSTEM IS UNBLOCKED utils.WriteOption("BACKUP_IS_WORKING", "0") @@ -909,6 +935,57 @@ func taskStopEdgeApp(args taskStopEdgeAppArgs) string { return string(resultJSON) } +func taskSetEdgeAppOptions(args taskSetEdgeAppOptionsArgs) string { + // Id is the edgeapp id + appID := args.ID + + + // Open the file to write the options, + // it is an env file in /home/system/components/apps//edgeapp.env + + // Get the path to the edgeapp.env file + edgeappEnvPath := "/home/system/components/apps/" + appID + "/edgeapp.env" + + // If the file does not exist, create it + if _, err := os.Stat(edgeappEnvPath); os.IsNotExist(err) { + // Create the file + _, err := os.Create(edgeappEnvPath) + if err != nil { + log.Printf("Error creating edgeapp.env file: %s", err) + } + } + + // It is an env file, so we can use go-dotenv to write the options + // Open the file + edgeappEnvFile, err := os.OpenFile(edgeappEnvPath, os.O_WRONLY, 0600) + if err != nil { + log.Printf("Error opening edgeapp.env file: %s", err) + } + + // Write the options to the file + for _, value := range args.Options { + // Write the option to the file + _, err := edgeappEnvFile.WriteString(value.Key + "=" + value.Value + "\n") + if err != nil { + log.Printf("Error writing option to edgeapp.env file: %s", err) + } + } + + // Close the file + err = edgeappEnvFile.Close() + if err != nil { + log.Printf("Error closing edgeapp.env file: %s", err) + } + + result := edgeapps.GetEdgeAppStatus(appID) + resultJSON, _ := json.Marshal(result) + + system.StartWs() + taskGetEdgeApps() // This task will imediatelly update the entry in the api database. + + return string(resultJSON) +} + func taskEnableOnline(args taskEnableOnlineArgs) string { fmt.Println("Executing taskEnableOnline for " + args.ID) @@ -961,6 +1038,35 @@ func taskSetReleaseVersion() string { return diagnostics.Version } +func taskUpdateSystemLoggerServices() string { + fmt.Println("Executing taskUpdateSystemLoggerServices") + // The input is an array of strings + // Each string is a service name to be logged + var input []string + + // Get the services + edgeAppsList := utils.ReadOption("EDGEAPPS_LIST") + var edgeApps []edgeapps.EdgeApp + err := json.Unmarshal([]byte(edgeAppsList), &edgeApps) + if err != nil { + log.Fatalf("failed to unmarshal EDGEAPPS_LIST: %v", err) + } + + for _, edgeApp := range edgeApps { + for _, service := range edgeApp.Services { + input = append(input, service.ID) + } + } + + input = append(input, "edgeboxctl") + input = append(input, "tunnel") + + // Run the system logger + system.UpdateSystemLoggerServices(input) + + return "{\"status\": \"ok\"}" +} + func taskGetEdgeApps() string { fmt.Println("Executing taskGetEdgeApps") diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 16f1d87..6d336f9 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -111,6 +111,8 @@ const ApiPath string = "apiPath" const EdgeAppsPath string = "edgeAppsPath" const EdgeAppsBackupPath string = "edgeAppsBackupPath" const WsPath string = "wsPath" +const LoggerPath string = "loggerPath" + // GetPath : Returns either the hardcoded path, or a overwritten value via .env file at project root. Register paths here for seamless working code between dev and prod environments ;) func GetPath(pathKey string) string { @@ -172,6 +174,13 @@ func GetPath(pathKey string) string { targetPath = "/home/system/components/ws/" } + case LoggerPath: + if env["LOGGER_PATH"] != "" { + targetPath = env["LOGGER_PATH"] + } else { + targetPath = "/home/system/components/logger/" + } + case BackupPasswordFileLocation: if env["BACKUP_PASSWORD_FILE_LOCATION"] != "" { From 5c736a1b854f1e5dfe3fbbcebaf02845a049af41 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Sun, 5 Nov 2023 02:01:16 +0100 Subject: [PATCH 15/20] Adjustments to cloud options, env location and expected vars --- internal/system/system.go | 12 ++++++++++++ internal/utils/utils.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/system/system.go b/internal/system/system.go index 42fe285..2d8d9bc 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -92,6 +92,18 @@ func SetupCloudOptions() { utils.WriteOption("EMAIL", cloudEnv["EMAIL"]) } + if cloudEnv["USERNAME"] != "" { + utils.WriteOption("USERNAME", cloudEnv["USERNAME"]) + } + + if cloudEnv["CLUSTER"] != "" { + utils.WriteOption("CLUSTER", cloudEnv["CLUSTER"]) + } + + if cloudEnv["CLUSTER_IP"] != "" { + utils.WriteOption("CLUSTER_IP", cloudEnv["CLUSTER_IP"]) + } + if cloudEnv["EDGEBOXIO_API_TOKEN"] != "" { utils.WriteOption("EDGEBOXIO_API_TOKEN", cloudEnv["EDGEBOXIO_API_TOKEN"]) } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 6d336f9..8b43984 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -132,7 +132,7 @@ func GetPath(pathKey string) string { if env["CLOUD_ENV_FILE_LOCATION"] != "" { targetPath = env["CLOUD_ENV_FILE_LOCATION"] } else { - targetPath = "/home/system/components/edgeboxctl/cloud.env" + targetPath = "/home/system/components/api/cloud.env" } case ApiEnvFileLocation: From f0047a7a0ce72f41dd8f4bc6cad5fe9babe913c1 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Sun, 5 Nov 2023 02:03:46 +0100 Subject: [PATCH 16/20] added cluster_ssh_port_var to read from env --- internal/system/system.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/system/system.go b/internal/system/system.go index 2d8d9bc..f4a58dd 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -104,6 +104,10 @@ func SetupCloudOptions() { utils.WriteOption("CLUSTER_IP", cloudEnv["CLUSTER_IP"]) } + if cloudEnv["CLUSTER_SSH_PORT"] != "" { + utils.WriteOption("CLUSTER_SSH_PORT", cloudEnv["CLUSTER_SSH_PORT"]) + } + if cloudEnv["EDGEBOXIO_API_TOKEN"] != "" { utils.WriteOption("EDGEBOXIO_API_TOKEN", cloudEnv["EDGEBOXIO_API_TOKEN"]) } From 319acbd0e1cf959795c2638d7991be690c0cd3af Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 5 Nov 2023 11:47:42 +0100 Subject: [PATCH 17/20] Add explicit support for prod amd64 build in Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index ca58644..03e3a19 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,10 @@ build-arm64: build-armhf: GOOS=linux GOARCH=arm RELEASE=prod make build +build-amd64: + GOOS=linux GOARCH=amd64 RELEASE=prod make build + + build: @echo "Building ${GOOS}-${GOARCH}" GOOS=${GOOS} GOARCH=${GOARCH} go build \ @@ -54,6 +58,7 @@ install-prod: build-prod install install-cloud: build-cloud install install-arm64: build-arm64 install install-armhf: build-armhf install +install-amd64: build-amd64 install start: systemctl start edgeboxctl From db565303112739520b3c1ae8bae881a7ee28473a Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 5 Nov 2023 12:19:09 +0100 Subject: [PATCH 18/20] Don't fail install if service cannot be stopped (does not exist yet) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 03e3a19..23e35ad 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ test-with-coverage: go test -tags=unit -timeout=600s -v ./... -coverprofile=coverage.out install: - sudo systemctl stop edgeboxctl + sudo systemctl stop edgeboxctl || true sudo rm -rf /usr/local/bin/edgeboxctl /usr/local/sbin/edgeboctl /lib/systemd/system/edgeboxctl.service sudo cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl sudo cp ./bin/edgeboxctl /usr/local/sbin/edgeboxctl From e238e9d71d5859c309c5ce9e5a2a08066e1951b4 Mon Sep 17 00:00:00 2001 From: Paulo Truta Date: Tue, 14 Nov 2023 21:00:09 +0100 Subject: [PATCH 19/20] Basic Auth and RemoveApp Support --- internal/edgeapps/edgeapps.go | 57 +++++++++++++++++- internal/tasks/tasks.go | 109 ++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/internal/edgeapps/edgeapps.go b/internal/edgeapps/edgeapps.go index 67bce80..f07b20c 100644 --- a/internal/edgeapps/edgeapps.go +++ b/internal/edgeapps/edgeapps.go @@ -25,6 +25,7 @@ type EdgeApp struct { InternetURL string `json:"internet_url"` Options []EdgeAppOption `json:"options"` NeedsConfig bool `json:"needs_config"` + Login EdgeAppLogin `json:"login"` } // MaybeEdgeApp : Boolean flag for validation of edgeapp existance @@ -55,11 +56,19 @@ type EdgeAppOption struct { IsInstallLocked bool `json:"is_install_locked"` } +type EdgeAppLogin struct { + Enabled bool `json:"enabled"` + Username string `json:"username"` + Password string `json:"password"` +} + const configFilename = "/edgebox-compose.yml" const envFilename = "/edgebox.env" const optionsTemplateFilename = "/edgeapp.template.env" const optionsEnvFilename = "/edgeapp.env" +const authEnvFilename = "/auth.env" const runnableFilename = "/.run" +const appdataFoldername = "/appdata" const myEdgeAppServiceEnvFilename = "/myedgeapp.env" const defaultContainerOperationSleepTime time.Duration = time.Second * 10 @@ -202,6 +211,21 @@ func GetEdgeApp(ID string) MaybeEdgeApp { } } + edgeAppBasicAuthEnabled := false + edgeAppBasicAuthUsername := "" + edgeAppBasicAuthPassword := "" + + edgeAppAuthEnv, err := godotenv.Read(utils.GetPath(utils.EdgeAppsPath) + ID + authEnvFilename) + if err != nil { + log.Println("No auth.env file found. Login status is disabled.") + } else { + if edgeAppAuthEnv["USERNAME"] != "" && edgeAppAuthEnv["PASSWORD"] != "" { + edgeAppBasicAuthEnabled = true + edgeAppBasicAuthUsername = edgeAppAuthEnv["USERNAME"] + edgeAppBasicAuthPassword = edgeAppAuthEnv["PASSWORD"] + } + } + result = MaybeEdgeApp{ EdgeApp: EdgeApp{ ID: ID, @@ -214,6 +238,8 @@ func GetEdgeApp(ID string) MaybeEdgeApp { InternetURL: edgeAppInternetURL, Options: edgeAppOptions, NeedsConfig: needsConfig, + Login: EdgeAppLogin{edgeAppBasicAuthEnabled, edgeAppBasicAuthUsername, edgeAppBasicAuthPassword}, + }, Valid: true, } @@ -268,11 +294,40 @@ func SetEdgeAppInstalled(ID string) bool { func SetEdgeAppNotInstalled(ID string) bool { + // Stop the app first + StopEdgeApp(ID) + + // Now remove any files result := true + err := os.Remove(utils.GetPath(utils.EdgeAppsPath) + ID + runnableFilename) if err != nil { result = false - log.Fatal(err) + log.Println(err) + } + + err = os.Remove(utils.GetPath(utils.EdgeAppsPath) + ID + authEnvFilename) + if err != nil { + result = false + log.Println(err) + } + + err = os.RemoveAll(utils.GetPath(utils.EdgeAppsPath) + ID + appdataFoldername) + if err != nil { + result = false + log.Println(err) + } + + err = os.Remove(utils.GetPath(utils.EdgeAppsPath) + ID + myEdgeAppServiceEnvFilename) + if err != nil { + result = false + log.Println(err) + } + + err = os.Remove(utils.GetPath(utils.EdgeAppsPath) + ID + optionsEnvFilename) + if err != nil { + result = false + log.Println(err) } buildFrameworkContainers() diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index e98720f..89fc848 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -39,6 +39,11 @@ type TaskOption struct { Value string `json:"value"` } +type TaskBasicAuth struct { + Username string `json:"username"` + Password string `json:"password"` +} + type taskSetupTunnelArgs struct { DomainName string `json:"domain_name"` } @@ -65,6 +70,15 @@ type taskSetEdgeAppOptionsArgs struct { Options []TaskOption `json:"options"` } +type taskSetEdgeAppBasicAuthArgs struct { + ID string `json:"id"` + Login TaskBasicAuth `json:"login"` +} + +type taskRemoveEdgeAppBasicAuthArgs struct { + ID string `json:"id"` +} + type taskEnableOnlineArgs struct { ID string `json:"id"` InternetURL string `json:"internet_url"` @@ -287,6 +301,30 @@ func ExecuteTask(task Task) Task { task.Result = sql.NullString{String: taskResult, Valid: true} } + case "set_edgeapp_basic_auth": + + log.Println("Settig EdgeApp Basic Authentication...") + var args taskSetEdgeAppBasicAuthArgs + err := json.Unmarshal([]byte(task.Args.String), &args) + if err != nil { + log.Printf("Error reading arguments of set_edgeapp_basic_auth task: %s", err) + } else { + taskResult := taskSetEdgeAppBasicAuth(args) + task.Result = sql.NullString{String: taskResult, Valid: true} + } + + case "remove_edgeapp_basic_auth": + + log.Println("Removing EdgeApp Basic Authentication...") + var args taskRemoveEdgeAppBasicAuthArgs + err := json.Unmarshal([]byte(task.Args.String), &args) + if err != nil { + log.Printf("Error reading arguments of remove_edgeapp_basic_auth task: %s", err) + } else { + taskResult := taskRemoveEdgeAppBasicAuth(args) + task.Result = sql.NullString{String: taskResult, Valid: true} + } + case "enable_online": log.Println("Enabling online access to EdgeApp...") @@ -986,6 +1024,77 @@ func taskSetEdgeAppOptions(args taskSetEdgeAppOptionsArgs) string { return string(resultJSON) } +func taskSetEdgeAppBasicAuth(args taskSetEdgeAppBasicAuthArgs) string { + // Id is the edgeapp id + appID := args.ID + + + // Open the file to write the options, + // it is an env file in /home/system/components/apps//auth.env + + // Get the path to the auth.env file + edgeappAuthEnvPath := "/home/system/components/apps/" + appID + "/auth.env" + + // If the file does not exist, create it + if _, err := os.Stat(edgeappAuthEnvPath); os.IsNotExist(err) { + // Create the file + _, err := os.Create(edgeappAuthEnvPath) + if err != nil { + log.Printf("Error creating auth.env file: %s", err) + } + } + + // It is an env file, so we can use go-dotenv to write the options + // Open the file + edgeappAuthEnvFile, err := os.OpenFile(edgeappAuthEnvPath, os.O_WRONLY, 0600) + if err != nil { + log.Printf("Error opening auth.env file: %s", err) + } + + // Write the login values to the file + _, err = edgeappAuthEnvFile.WriteString("USERNAME=" + args.Login.Username + "\n" + "PASSWORD=" + args.Login.Password + "\n") + if err != nil { + log.Printf("Error writing credentials to auth.env file: %s", err) + } + + // Close the file + err = edgeappAuthEnvFile.Close() + if err != nil { + log.Printf("Error closing auth.env file: %s", err) + } + + result := edgeapps.GetEdgeAppStatus(appID) + resultJSON, _ := json.Marshal(result) + + system.StartWs() + taskGetEdgeApps() // This task will imediatelly update the entry in the api database. + + return string(resultJSON) +} + +func taskRemoveEdgeAppBasicAuth(args taskRemoveEdgeAppBasicAuthArgs) string { + // Id is the edgeapp id + appID := args.ID + + // Get the path to the auth.env file + edgeappAuthEnvFile := "/auth.env" + + fmt.Println("Removing auth.env file" + edgeappAuthEnvFile) + + err := os.Remove(utils.GetPath(utils.EdgeAppsPath) + args.ID + edgeappAuthEnvFile) + if err != nil { + log.Fatal(err) + } + + result := edgeapps.GetEdgeAppStatus(appID) + resultJSON, _ := json.Marshal(result) + + system.StartWs() + taskGetEdgeApps() // This task will imediatelly update the entry in the api database. + + return string(resultJSON) +} + func taskEnableOnline(args taskEnableOnlineArgs) string { fmt.Println("Executing taskEnableOnline for " + args.ID) From 4e399f63ea10c750d02098da9985840aee4f1296 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Nov 2023 12:31:08 +0100 Subject: [PATCH 20/20] Bump actions/setup-go from 3 to 4 (#31) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 45ae769..407d3f1 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ^1.15 - name: Check out code