Backups feature
Paulo Truta
parent
f7823b9e20
commit
a66ff40e65
5
Makefile
5
Makefile
|
|
@ -21,9 +21,6 @@ build-arm64:
|
|||
build-armhf:
|
||||
GOOS=linux GOARCH=arm RELEASE=prod make build
|
||||
|
||||
build-amd64:
|
||||
GOOS=linux GOARCH=amd64 RELEASE=prod make build
|
||||
|
||||
build:
|
||||
@echo "Building ${GOOS}-${GOARCH}"
|
||||
GOOS=${GOOS} GOARCH=${GOARCH} go build \
|
||||
|
|
@ -52,7 +49,7 @@ install-cloud: build-cloud
|
|||
@echo "To start edgeboxctl run: systemctl start edgeboxctl"
|
||||
|
||||
install-prod: build-prod
|
||||
-sudo systemctl stop edgeboxctl
|
||||
sudo systemctl stop edgeboxctl
|
||||
sudo rm -rf /usr/local/bin/edgeboxctl /lib/systemd/system/edgeboxctl.service
|
||||
sudo cp ./bin/edgeboxctl /usr/local/bin/edgeboxctl
|
||||
sudo cp ./edgeboxctl.service /lib/systemd/system/edgeboxctl.service
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
package backups
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/edgebox-iot/edgeboxctl/internal/diagnostics"
|
||||
"github.com/edgebox-iot/edgeboxctl/internal/utils"
|
||||
"github.com/shirou/gopsutil/disk"
|
||||
)
|
||||
|
||||
// Repository : Struct representing the backup repository of a device in the system
|
||||
type Repository struct {
|
||||
ID string `json:"id"`
|
||||
FileCount int64 `json:"file_count"`
|
||||
Size string `json:"size"`
|
||||
Snapshots []Snapshot `json:"snapshots"`
|
||||
Status string `json:"status"`
|
||||
UsageStat UsageStat `json:"usage_stat"`
|
||||
}
|
||||
|
||||
// Snapshot : Struct representing a single snapshot in the backup repository
|
||||
type Snapshot struct {
|
||||
ID string `json:"id"`
|
||||
time string `json:"time"`
|
||||
}
|
||||
|
|
@ -130,6 +130,22 @@ func GetServiceStatus(serviceID string) string {
|
|||
return utils.Exec(wsPath, "systemctl", cmdargs)
|
||||
}
|
||||
|
||||
func CreateBackupsPasswordFile(password string) {
|
||||
// Create a password file for backups
|
||||
backupPasswordFile := utils.GetPath(utils.BackupPasswordFileLocation)
|
||||
backupPasswordFileDir := filepath.Dir(backupPasswordFile)
|
||||
|
||||
if _, err := os.Stat(backupPasswordFileDir); os.IsNotExist(err) {
|
||||
os.MkdirAll(backupPasswordFileDir, 0755)
|
||||
}
|
||||
|
||||
// Write the password to the file, overriting an existing file
|
||||
err := ioutil.WriteFile(backupPasswordFile, []byte(password), 0644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// CreateTunnel: Creates a tunnel via cloudflared, needs to be authenticated first
|
||||
func CreateTunnel(configDestination string) {
|
||||
fmt.Println("Creating Tunnel for Edgebox.")
|
||||
|
|
|
|||
|
|
@ -66,6 +66,14 @@ type taskEnablePublicDashboardArgs struct {
|
|||
InternetURL string `json:"internet_url"`
|
||||
}
|
||||
|
||||
type taskSetupBackupsArgs struct {
|
||||
Service string `json:"service"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
RepositoryName string `json:"repository_name"`
|
||||
RepositoryPassword string `json:"repository_password"`
|
||||
}
|
||||
|
||||
|
||||
const STATUS_CREATED int = 0
|
||||
const STATUS_EXECUTING int = 1
|
||||
|
|
@ -124,6 +132,7 @@ func ExecuteTask(task Task) Task {
|
|||
|
||||
formatedDatetime := utils.GetSQLiteFormattedDateTime(time.Now())
|
||||
|
||||
fmt.Println("Changing task status to executing: " + task.Task)
|
||||
_, err = statement.Exec(STATUS_EXECUTING, formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
|
|
@ -135,6 +144,34 @@ func ExecuteTask(task Task) Task {
|
|||
log.Println("Task: " + task.Task)
|
||||
log.Println("Args: " + task.Args.String)
|
||||
switch task.Task {
|
||||
case "setup_backups":
|
||||
|
||||
log.Println("Setting up Backups Destination...")
|
||||
var args taskSetupBackupsArgs
|
||||
err := json.Unmarshal([]byte(task.Args.String), &args)
|
||||
if err != nil {
|
||||
log.Println("Error reading arguments of setup_backups task: %s", err)
|
||||
} else {
|
||||
taskResult := taskSetupBackups(args)
|
||||
taskResultBool := true
|
||||
// Check if returned taskResult string contains "error"
|
||||
if strings.Contains(taskResult, "error") {
|
||||
taskResultBool = false
|
||||
}
|
||||
task.Result = sql.NullString{String: taskResult, Valid: taskResultBool}
|
||||
}
|
||||
|
||||
case "start_backup":
|
||||
|
||||
log.Println("Backing up Edgebox...")
|
||||
taskResult := taskBackup()
|
||||
taskResultBool := true
|
||||
// Check if returned taskResult string contains "error"
|
||||
if strings.Contains(taskResult, "error") {
|
||||
taskResultBool = false
|
||||
}
|
||||
task.Result = sql.NullString{String: taskResult, Valid: taskResultBool}
|
||||
|
||||
case "setup_tunnel":
|
||||
|
||||
log.Println("Setting up Cloudflare Tunnel...")
|
||||
|
|
@ -272,11 +309,13 @@ func ExecuteTask(task Task) Task {
|
|||
formatedDatetime = utils.GetSQLiteFormattedDateTime(time.Now())
|
||||
|
||||
if task.Result.Valid {
|
||||
fmt.Println("Task Result: " + task.Result.String)
|
||||
_, err = statement.Exec(STATUS_FINISHED, task.Result.String, formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement with result info
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Error executing task with result: " + task.Result.String)
|
||||
_, err = statement.Exec(STATUS_ERROR, "Error", formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement with Error info
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
|
|
@ -338,6 +377,8 @@ func ExecuteSchedules(tick int) {
|
|||
if tick%30 == 0 {
|
||||
// Executing every 30 ticks
|
||||
log.Println(taskGetEdgeApps())
|
||||
// RESET SOME VARIABLES HERE IF NEEDED, SINCE SYSTEM IS UNBLOCKED
|
||||
utils.WriteOption("BACKUP_IS_WORKING", "0")
|
||||
}
|
||||
|
||||
if tick%60 == 0 {
|
||||
|
|
@ -345,10 +386,223 @@ func ExecuteSchedules(tick int) {
|
|||
log.Println("System IP is: " + ip)
|
||||
}
|
||||
|
||||
if tick%3600 == 0 {
|
||||
// Executing every 3600 ticks (1 hour)
|
||||
backup := taskAutoBackup()
|
||||
log.Println("Auto Backup: " + backup)
|
||||
}
|
||||
|
||||
// Just add a schedule here if you need a custom one (every "tick hour", every "tick day", etc...)
|
||||
|
||||
}
|
||||
|
||||
func taskSetupBackups(args taskSetupBackupsArgs) string {
|
||||
fmt.Println("Executing taskSetupBackups" + args.Service)
|
||||
// ...
|
||||
service_url := ""
|
||||
key_id_name := "AWS_ACCESS_KEY_ID"
|
||||
key_secret_name := "AWS_SECRET_ACCESS_KEY"
|
||||
repo_location := "/home/system/components/apps/"
|
||||
service_found := false
|
||||
|
||||
switch args.Service {
|
||||
case "s3":
|
||||
service_url = "s3.amazonaws.com/"
|
||||
service_found = true
|
||||
case "b2":
|
||||
service_url = ""
|
||||
key_id_name = "B2_ACCOUNT_ID"
|
||||
key_secret_name = "B2_ACCOUNT_KEY"
|
||||
service_found = true
|
||||
case "wasabi":
|
||||
service_found = true
|
||||
service_url = "s3.wasabisys.com/"
|
||||
}
|
||||
|
||||
if !service_found {
|
||||
fmt.Println("Service not found")
|
||||
return "{\"status\": \"error\", \"message\": \"Service not found\"}"
|
||||
}
|
||||
|
||||
fmt.Println("Creating env vars for authentication with backup service")
|
||||
os.Setenv(key_id_name, args.AccessKeyID)
|
||||
os.Setenv(key_secret_name, args.SecretAccessKey)
|
||||
|
||||
fmt.Println("Creating restic password file")
|
||||
|
||||
system.CreateBackupsPasswordFile(args.RepositoryPassword)
|
||||
|
||||
fmt.Println("Initializing restic repository")
|
||||
utils.WriteOption("BACKUP_IS_WORKING", "1")
|
||||
|
||||
cmdArgs := []string{"-r", args.Service + ":" + service_url + args.RepositoryName + ":" + repo_location, "init", "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"}
|
||||
|
||||
result := utils.ExecAndStream(repo_location, "restic", cmdArgs)
|
||||
|
||||
utils.WriteOption("BACKUP_IS_WORKING", "0")
|
||||
|
||||
// See if result contains the substring "Fatal:"
|
||||
if strings.Contains(result, "Fatal:") {
|
||||
fmt.Println("Error initializing restic repository")
|
||||
|
||||
utils.WriteOption("BACKUP_STATUS", "error")
|
||||
utils.WriteOption("BACKUP_ERROR_MESSAGE", result)
|
||||
|
||||
return "{\"status\": \"error\", \"message\": \"" + result + "\"}"
|
||||
}
|
||||
|
||||
// Save options to database
|
||||
utils.WriteOption("BACKUP_STATUS", "initiated")
|
||||
utils.WriteOption("BACKUP_SERVICE", args.Service)
|
||||
utils.WriteOption("BACKUP_SERVICE_URL", service_url)
|
||||
utils.WriteOption("BACKUP_REPOSITORY_NAME", args.RepositoryName)
|
||||
utils.WriteOption("BACKUP_REPOSITORY_PASSWORD", args.RepositoryPassword)
|
||||
utils.WriteOption("BACKUP_REPOSITORY_ACCESS_KEY_ID", args.AccessKeyID)
|
||||
utils.WriteOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY", args.SecretAccessKey)
|
||||
utils.WriteOption("BACKUP_REPOSITORY_LOCATION", repo_location)
|
||||
|
||||
// Populate Stats right away
|
||||
taskGetBackupStatus()
|
||||
|
||||
return "{\"status\": \"ok\"}"
|
||||
|
||||
}
|
||||
|
||||
func taskRemoveBackups() string {
|
||||
|
||||
fmt.Println("Executing taskRemoveBackups")
|
||||
|
||||
// ... This deletes the restic repository
|
||||
// cmdArgs := []string{"-r", "s3:https://s3.amazonaws.com/edgebox-backups:/home/system/components/apps/", "forget", "latest", "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"}
|
||||
|
||||
utils.WriteOption("BACKUP_STATUS", "")
|
||||
utils.WriteOption("BACKUP_IS_WORKING", "0")
|
||||
|
||||
return "{\"status\": \"ok\"}"
|
||||
|
||||
}
|
||||
|
||||
func taskBackup() string {
|
||||
fmt.Println("Executing taskBackup")
|
||||
|
||||
// Load Backup Options
|
||||
backup_service := utils.ReadOption("BACKUP_SERVICE")
|
||||
backup_service_url := utils.ReadOption("BACKUP_SERVICE_URL")
|
||||
backup_repository_name := utils.ReadOption("BACKUP_REPOSITORY_NAME")
|
||||
// backup_repository_password := utils.ReadOption("BACKUP_REPOSITORY_PASSWORD")
|
||||
backup_repository_access_key_id := utils.ReadOption("BACKUP_REPOSITORY_ACCESS_KEY_ID")
|
||||
backup_repository_secret_access_key := utils.ReadOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY")
|
||||
backup_repository_location := utils.ReadOption("BACKUP_REPOSITORY_LOCATION")
|
||||
|
||||
key_id_name := "AWS_ACCESS_KEY_ID"
|
||||
key_secret_name := "AWS_SECRET_ACCESS_KEY"
|
||||
service_found := false
|
||||
|
||||
switch backup_service {
|
||||
case "s3":
|
||||
service_found = true
|
||||
case "b2":
|
||||
key_id_name = "B2_ACCOUNT_ID"
|
||||
key_secret_name = "B2_ACCOUNT_KEY"
|
||||
service_found = true
|
||||
case "wasabi":
|
||||
service_found = true
|
||||
}
|
||||
|
||||
if !service_found {
|
||||
fmt.Println("Service not found")
|
||||
return "{\"status\": \"error\", \"message\": \"Backup Service not found\"}"
|
||||
}
|
||||
|
||||
fmt.Println("Creating env vars for authentication with backup service")
|
||||
fmt.Println(key_id_name)
|
||||
os.Setenv(key_id_name, backup_repository_access_key_id)
|
||||
fmt.Println(key_secret_name)
|
||||
os.Setenv(key_secret_name, backup_repository_secret_access_key)
|
||||
|
||||
|
||||
utils.WriteOption("BACKUP_IS_WORKING", "1")
|
||||
|
||||
// ... This backs up the restic repository
|
||||
cmdArgs := []string{"-r", backup_service + ":" + backup_service_url + backup_repository_name + ":" + backup_repository_location, "backup", backup_repository_location, "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"}
|
||||
result := utils.ExecAndStream(backup_repository_location, "restic", cmdArgs)
|
||||
|
||||
utils.WriteOption("BACKUP_IS_WORKING", "0")
|
||||
// Write as Unix timestamp
|
||||
utils.WriteOption("BACKUP_LAST_RUN", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
|
||||
// See if result contains the substring "Fatal:"
|
||||
if strings.Contains(result, "Fatal:") {
|
||||
fmt.Println("Error backing up")
|
||||
utils.WriteOption("BACKUP_STATUS", "error")
|
||||
utils.WriteOption("BACKUP_ERROR_MESSAGE", result)
|
||||
return "{\"status\": \"error\", \"message\": \"" + result + "\"}"
|
||||
}
|
||||
|
||||
utils.WriteOption("BACKUP_STATUS", "working")
|
||||
taskGetBackupStatus()
|
||||
return "{\"status\": \"ok\"}"
|
||||
|
||||
}
|
||||
|
||||
func taskAutoBackup() string {
|
||||
fmt.Println("Executing taskAutoBackup")
|
||||
|
||||
// Get Backup Status
|
||||
backup_status := utils.ReadOption("BACKUP_STATUS")
|
||||
// We only backup is the status is "working"
|
||||
if backup_status == "working" {
|
||||
return taskBackup()
|
||||
} else {
|
||||
fmt.Println("Backup status is not working... skipping")
|
||||
return "{\"status\": \"skipped\"}"
|
||||
}
|
||||
}
|
||||
|
||||
func taskGetBackupStatus() string {
|
||||
fmt.Println("Executing taskGetBackupStatus")
|
||||
|
||||
// Load Backup Options
|
||||
backup_service := utils.ReadOption("BACKUP_SERVICE")
|
||||
backup_service_url := utils.ReadOption("BACKUP_SERVICE_URL")
|
||||
backup_repository_name := utils.ReadOption("BACKUP_REPOSITORY_NAME")
|
||||
// backup_repository_password := utils.ReadOption("BACKUP_REPOSITORY_PASSWORD")
|
||||
backup_repository_access_key_id := utils.ReadOption("BACKUP_REPOSITORY_ACCESS_KEY_ID")
|
||||
backup_repository_secret_access_key := utils.ReadOption("BACKUP_REPOSITORY_SECRET_ACCESS_KEY")
|
||||
backup_repository_location := utils.ReadOption("BACKUP_REPOSITORY_LOCATION")
|
||||
|
||||
key_id_name := "AWS_ACCESS_KEY_ID"
|
||||
key_secret_name := "AWS_SECRET_ACCESS_KEY"
|
||||
service_found := false
|
||||
|
||||
switch backup_service {
|
||||
case "s3":
|
||||
service_found = true
|
||||
case "b2":
|
||||
key_id_name = "B2_ACCOUNT_ID"
|
||||
key_secret_name = "B2_ACCOUNT_KEY"
|
||||
service_found = true
|
||||
case "wasabi":
|
||||
service_found = true
|
||||
}
|
||||
|
||||
if !service_found {
|
||||
fmt.Println("Service not found")
|
||||
return "{\"status\": \"error\", \"message\": \"Backup Service not found\"}"
|
||||
}
|
||||
|
||||
fmt.Println("Creating env vars for authentication with backup service")
|
||||
os.Setenv(key_id_name, backup_repository_access_key_id)
|
||||
os.Setenv(key_secret_name, backup_repository_secret_access_key)
|
||||
|
||||
// ... This gets the restic repository status
|
||||
cmdArgs := []string{"-r", backup_service + ":" + backup_service_url + backup_repository_name + ":" + backup_repository_location, "stats", "--password-file", utils.GetPath(utils.BackupPasswordFileLocation), "--verbose=3"}
|
||||
utils.WriteOption("BACKUP_STATS", utils.ExecAndStream(backup_repository_location, "restic", cmdArgs))
|
||||
|
||||
return "{\"status\": \"ok\"}"
|
||||
|
||||
}
|
||||
|
||||
func taskSetupTunnel(args taskSetupTunnelArgs) string {
|
||||
fmt.Println("Executing taskSetupTunnel")
|
||||
wsPath := utils.GetPath(utils.WsPath)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import (
|
|||
)
|
||||
|
||||
// ExecAndStream : Runs a terminal command, but streams progress instead of outputting. Ideal for long lived process that need to be logged.
|
||||
func ExecAndStream(path string, command string, args []string) {
|
||||
func ExecAndStream(path string, command string, args []string) string {
|
||||
|
||||
cmd := exec.Command(command, args...)
|
||||
|
||||
|
|
@ -27,13 +27,17 @@ func ExecAndStream(path string, command string, args []string) {
|
|||
|
||||
err := cmd.Run()
|
||||
|
||||
outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes())
|
||||
|
||||
returnVal := outStr
|
||||
if err != nil {
|
||||
fmt.Printf("cmd.Run() failed with %s\n", err)
|
||||
returnVal = errStr
|
||||
}
|
||||
|
||||
outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes())
|
||||
fmt.Printf("\nout:\n%s\nerr:\n%s\n", outStr, errStr)
|
||||
|
||||
return returnVal
|
||||
}
|
||||
|
||||
// Exec : Runs a terminal Command, catches and logs errors, returns the result.
|
||||
|
|
@ -100,6 +104,7 @@ func GetSQLiteFormattedDateTime(t time.Time) string {
|
|||
return formatedDatetime
|
||||
}
|
||||
|
||||
const BackupPasswordFileLocation string = "backupPasswordFileLocation"
|
||||
const CloudEnvFileLocation string = "cloudEnvFileLocation"
|
||||
const ApiEnvFileLocation string = "apiEnvFileLocation"
|
||||
const ApiPath string = "apiPath"
|
||||
|
|
@ -159,6 +164,14 @@ func GetPath(pathKey string) string {
|
|||
targetPath = "/home/system/components/ws/"
|
||||
}
|
||||
|
||||
case BackupPasswordFileLocation:
|
||||
|
||||
if env["BACKUP_PASSWORD_FILE_LOCATION"] != "" {
|
||||
targetPath = env["BACKUP_PASSWORD_FILE_LOCATION"]
|
||||
} else {
|
||||
targetPath = "/home/system/components/backups/pw.txt"
|
||||
}
|
||||
|
||||
default:
|
||||
|
||||
log.Printf("path_key %s nonexistant in GetPath().\n", pathKey)
|
||||
|
|
@ -207,7 +220,7 @@ func ReadOption(optionKey string) string {
|
|||
err = db.QueryRow("SELECT value FROM option WHERE name = ?", optionKey).Scan(&optionValue)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
log.Println(err.Error())
|
||||
}
|
||||
|
||||
db.Close()
|
||||
|
|
|
|||
Loading…
Reference in New Issue