Merge fc6c5c65c8 into b11034dd17

Co-authored-by: = <=>

Makefile

@@ -69,5 +69,8 @@ start:
 stop:
 	systemctl stop edgeboxctl
 
+status:
+	systemctl status edgeboxctl
+
 log: start
 	journalctl -fu edgeboxctl

internal/edgeapps/edgeapps.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/edgebox-iot/edgeboxctl/internal/system"
 	"github.com/edgebox-iot/edgeboxctl/internal/utils"
+	"github.com/edgebox-iot/edgeboxctl/internal/diagnostics"
 )
 
 // EdgeApp : Struct representing an EdgeApp in the system
@@ -18,6 +19,7 @@ type EdgeApp struct {
 	ID                 string           `json:"id"`
 	Name               string           `json:"name"`
 	Description        string           `json:"description"`
+	Experimental	   bool             `json:"experimental"`
 	Status             EdgeAppStatus    `json:"status"`
 	Services           []EdgeAppService `json:"services"`
 	InternetAccessible bool             `json:"internet_accessible"`
@@ -69,6 +71,7 @@ const optionsEnvFilename = "/edgeapp.env"
 const authEnvFilename = "/auth.env"
 const runnableFilename = "/.run"
 const appdataFoldername = "/appdata"
+const postInstallFilename = "/edgebox-postinstall.done"
 const myEdgeAppServiceEnvFilename = "/myedgeapp.env"
 const defaultContainerOperationSleepTime time.Duration = time.Second * 10
 
@@ -86,6 +89,7 @@ func GetEdgeApp(ID string) MaybeEdgeApp {
 
 		edgeAppName := ID
 		edgeAppDescription := ""
+		edgeAppExperimental := false
 		edgeAppOptions := []EdgeAppOption{}
 
 		edgeAppEnv, err := godotenv.Read(utils.GetPath(utils.EdgeAppsPath) + ID + envFilename)
@@ -99,6 +103,9 @@ func GetEdgeApp(ID string) MaybeEdgeApp {
 			if edgeAppEnv["EDGEAPP_DESCRIPTION"] != "" {
 				edgeAppDescription = edgeAppEnv["EDGEAPP_DESCRIPTION"]
 			}
+			if edgeAppEnv["EDGEAPP_EXPERIMENTAL"] == "true" {
+				edgeAppExperimental = true
+			}
 		}
 
 		needsConfig := false
@@ -231,6 +238,7 @@ func GetEdgeApp(ID string) MaybeEdgeApp {
 				ID:                 ID,
 				Name:               edgeAppName,
 				Description:        edgeAppDescription,
+				Experimental:       edgeAppExperimental,
 				Status:             GetEdgeAppStatus(ID),
 				Services:           GetEdgeAppServices(ID),
 				InternetAccessible: edgeAppInternetAccessible,
@@ -263,22 +271,54 @@ func IsEdgeAppInstalled(ID string) bool {
 
 }
 
+func writeAppRunnableFiles(ID string) bool {
+	edgeAppPath := utils.GetPath(utils.EdgeAppsPath)
+	_, err := os.Stat(edgeAppPath + ID + runnableFilename)
+	if os.IsNotExist(err) {
+		_, err := os.Create(edgeAppPath + ID + runnableFilename)
+		if err != nil {
+			log.Fatal("Runnable file for EdgeApp could not be created!")
+			return false
+		}
+
+		// Check the block default apps option
+        blockDefaultAppsOption := utils.ReadOption("DASHBOARD_BLOCK_DEFAULT_APPS_PUBLIC_ACCESS")
+        if blockDefaultAppsOption != "yes" {
+            // Create myedgeapp.env file with default network URL
+            envFilePath := edgeAppPath + ID + myEdgeAppServiceEnvFilename
+            
+			var networkURL string
+			domainName := utils.ReadOption("DOMAIN_NAME")
+
+			if domainName != "" {
+				networkURL = ID + "." + domainName
+			} else if diagnostics.GetReleaseVersion() == diagnostics.CLOUD_VERSION {
+				cluster := utils.ReadOption("CLUSTER") 
+				username := utils.ReadOption("USERNAME")
+				if cluster != "" && username != "" {
+					networkURL = username + "-" + ID + "." + cluster
+				}
+			} else {
+				networkURL = ID + "." + system.GetHostname() + ".local" // default 
+			}
+			
+            env, _ := godotenv.Unmarshal("INTERNET_URL=" + networkURL)
+            err = godotenv.Write(env, envFilePath)
+            if err != nil {
+                log.Printf("Error creating myedgeapp.env file: %s", err)
+                // result = false
+            }
+        }
+	}
+	return true
+}
+
 func SetEdgeAppInstalled(ID string) bool {
 
 	result := true
-	edgeAppPath := utils.GetPath(utils.EdgeAppsPath)
-
-	_, err := os.Stat(edgeAppPath + ID + runnableFilename)
-	if os.IsNotExist(err) {
-
-		_, err := os.Create(edgeAppPath + ID + runnableFilename)
-		result = true
-
-		if err != nil {
-			log.Fatal("Runnable file for EdgeApp could not be created!")
-			result = false
-		}
 
+	if writeAppRunnableFiles(ID) {
+		
 		buildFrameworkContainers()
 
 	} else {
@@ -292,6 +332,21 @@ func SetEdgeAppInstalled(ID string) bool {
 
 }
 
+func SetEdgeAppBulkInstalled(IDs []string) bool {
+
+	result := true
+
+	for _, ID := range IDs {
+		writeAppRunnableFiles(ID)
+	}
+
+	buildFrameworkContainers()
+
+	return result
+
+}
+
+
 func SetEdgeAppNotInstalled(ID string) bool {
 
 	// Stop the app first
@@ -330,6 +385,12 @@ func SetEdgeAppNotInstalled(ID string) bool {
 		log.Println(err)
 	}
 
+	err = os.Remove(utils.GetPath(utils.EdgeAppsPath) + ID + postInstallFilename)
+	if err != nil {
+		result = false
+		log.Println(err)
+	}
+
 	buildFrameworkContainers()
 
 	return result
@@ -416,12 +477,24 @@ func GetEdgeAppServices(ID string) []EdgeAppService {
 	var edgeAppServices []EdgeAppService
 
 	for _, serviceID := range serviceSlices {
-		cmdArgs = []string{"-f", wsPath + "/docker-compose.yml", "exec", "-T", serviceID, "echo", "'Service Check'"}
+		shouldBeRunning := false
-		cmdResult := utils.Exec(wsPath, "docker-compose", cmdArgs)
 		isRunning := false
-		if cmdResult != "" {
+
-			isRunning = true
+		// Is service "runnable" when .run lockfile in the app folder
+		_, err := os.Stat(utils.GetPath(utils.EdgeAppsPath) + ID + runnableFilename)
+		if !os.IsNotExist(err) {
+			shouldBeRunning = true
 		}
+
+		// Check if the service is actually running
+		if shouldBeRunning {
+			cmdArgs = []string{"-f", wsPath + "/docker-compose.yml", "exec", "-T", serviceID, "echo", "'Service Check'"}
+			cmdResult := utils.Exec(wsPath, "docker-compose", cmdArgs)
+			if cmdResult != "" {
+				isRunning = true
+			}
+		}
+
 		edgeAppServices = append(edgeAppServices, EdgeAppService{ID: serviceID, IsRunning: isRunning})
 	}
 

internal/storage/storage.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"strconv"
 	"os"
 	"path/filepath"
 	"strings"
@@ -73,12 +74,13 @@ const (
 	DISK_TYPE_SDA   DeviceIdentifier = "sda"
 	DISK_TYPE_MCBLK DeviceIdentifier = "mmcblk0"
 	DISK_TYPE_VDA   DeviceIdentifier = "vda"
+	MIN_DISK_SIZE   int              = 1048576 // 1GB in bytes
 )
 
 func GetDeviceIdentifier(release_version diagnostics.ReleaseVersion) DeviceIdentifier {
 	switch release_version {
 	case diagnostics.CLOUD_VERSION:
-		return DISK_TYPE_VDA
+		return DISK_TYPE_SDA
 	case diagnostics.PROD_VERSION:
 		return DISK_TYPE_MCBLK
 	}
@@ -134,7 +136,13 @@ func GetDevices(release_version diagnostics.ReleaseVersion) []Device {
 				currentDevice.InUse = currentDeviceInUseFlag
 				currentDeviceInUseFlag = false
 				currentPartitions = []Partition{}
-				devices = append(devices, currentDevice)
+				size, err := strconv.Atoi(currentDevice.Size)
+				if err != nil {
+					size = 0
+				}
+				if size > MIN_DISK_SIZE {
+					devices = append(devices, currentDevice)
+				}
 			} else {
 				firstDevice = false
 			}
@@ -193,8 +201,22 @@ func GetDevices(release_version diagnostics.ReleaseVersion) []Device {
 		currentDevice.Status.Description = "Not configured"
 	}
 	currentDevice.InUse = currentDeviceInUseFlag
-	devices = append([]Device{currentDevice}, devices...) // Prepending the first device...
 
+	fmt.Println("Secondary Storage Devices Found: ", len(devices))
+	fmt.Println("Main Storage Device size: ", currentDevice.Size)
+
+	// only append device if size > 1GB
+	if currentDevice.Size != "" && currentDevice.Size != "0" {
+		// Convert size to int
+		// Convert string to int
+		size, err := strconv.Atoi(currentDevice.Size)
+		if err != nil {
+			size = 0
+		}
+		if size > MIN_DISK_SIZE {
+			devices = append([]Device{currentDevice}, devices...) // Prepending the first device...
+		}
+	}
 	devices = getDevicesSpaceUsage(devices)
 
 	return devices

internal/system/system.go

@@ -425,3 +425,94 @@ func CopyFile(src string, dest string) error {
     return nil
 }
 
+func CheckUpdates() {
+	fmt.Println("Checking for Edgebox System Updates.")
+	
+	// Configure the service and start it
+	cmd := exec.Command("sh", "/home/system/components/updater/run.sh", "--check")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		panic(err)
+	}
+	scanner := bufio.NewScanner(stdout)
+	err = cmd.Start()
+	if err != nil {
+		panic(err)
+	}
+	for scanner.Scan() {
+		// fmt.Println(scanner.Text())
+		text := scanner.Text()
+		fmt.Println(text)
+	}
+	if scanner.Err() != nil {
+		cmd.Process.Kill()
+		cmd.Wait()
+		fmt.Println("Error running updates check.")
+		utils.WriteOption("SYSTEM_UPDATES", "[]")
+		return
+	}
+
+	// Read targets.env file into JSON list structure
+	targets := []string{}
+	targetsFile, err := os.Open("/home/system/components/updater/targets.env")
+	if err != nil {
+		fmt.Println("No targets.env file found. Skipping.")
+		utils.WriteOption("SYSTEM_UPDATES", "[]")
+		return
+	}
+	defer targetsFile.Close()
+	scanner = bufio.NewScanner(targetsFile)
+	for scanner.Scan() {
+		text := scanner.Text()
+		// text line should look like: {"target": "<target>", "version": "<version>"}
+		target := strings.Split(text, "=")
+		newText := "{\"target\": \"" + strings.Replace(target[0], "_VERSION", "", -1) + "\", \"version\": \"" + target[1] + "\"}"
+		targets = append(targets, newText)
+	}
+	if scanner.Err() != nil {
+		fmt.Println("Error reading update targets file.")
+		utils.WriteOption("SYSTEM_UPDATES", "[]")
+		return
+	}
+
+	// convert targets to string
+	targetsString := strings.Join(targets, ",")
+	targetsString = "[" + targetsString + "]"
+
+	fmt.Println(targetsString)
+
+	// Write option with targets
+	utils.WriteOption("SYSTEM_UPDATES", targetsString)
+}
+
+func ApplyUpdates() {
+	fmt.Println("Applying Edgebox System Updates.")
+
+	utils.WriteOption("UPDATING_SYSTEM", "true")
+	
+	// Configure the service and start it
+	cmd := exec.Command("sh", "/home/system/components/updater/run.sh", "--update")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		panic(err)
+	}
+	scanner := bufio.NewScanner(stdout)
+	err = cmd.Start()
+	if err != nil {
+		panic(err)
+	}
+	for scanner.Scan() {
+		fmt.Println(scanner.Text())
+		text := scanner.Text()
+		fmt.Println(text)
+	}
+	if scanner.Err() != nil {
+		cmd.Process.Kill()
+		cmd.Wait()
+		panic(scanner.Err())
+	}
+
+	// If the system did not yet restart, set updating system to false
+	utils.WriteOption("UPDATING_SYSTEM", "false")
+}
+

internal/tasks/tasks.go

@@ -56,6 +56,10 @@ type taskInstallEdgeAppArgs struct {
 	ID string `json:"id"`
 }
 
+type taskInstallBulkEdgeAppsArgs struct {
+	IDS []string `json:"ids"`
+}
+
 type taskRemoveEdgeAppArgs struct {
 	ID string `json:"id"`
 }
@@ -100,6 +104,10 @@ type taskSetupBackupsArgs struct {
 	RepositoryPassword string `json:"repository_password"`
 }
 
+type taskStartShellArgs struct {
+	Timeout int `json:"timeout"`
+}
+
 
 const STATUS_CREATED int = 0
 const STATUS_EXECUTING int = 1
@@ -142,6 +150,33 @@ func GetNextTask() Task {
 
 }
 
+// GetExecutingTasks : Performs a MySQL query over the device's Edgebox API to obtain all tasks that are currently executing
+func GetExecutingTasks() []Task {
+	// Will try to connect to API database, which should be running locally under WS.
+	db, err := sql.Open("sqlite3", utils.GetSQLiteDbConnectionDetails())
+	if err != nil {
+		panic(err.Error())
+	}
+	results, err := db.Query("SELECT id, task, args, status, result, created, updated FROM task WHERE status = 1;")
+	if err != nil {
+		panic(err.Error())
+	}
+
+	var tasks []Task
+	for results.Next() {
+		// for each row, scan the result into our task composite object
+		var task Task
+		err = results.Scan(&task.ID, &task.Task, &task.Args, &task.Status, &task.Result, &task.Created, &task.Updated)
+		if err != nil {
+			panic(err.Error()) // proper error handling instead of panic in your app
+		}
+		tasks = append(tasks, task)
+	}
+	results.Close()
+	db.Close()
+	return tasks
+}
+
 // ExecuteTask : Performs execution of the given task, updating the task status as it goes, and publishing the task result
 func ExecuteTask(task Task) Task {
 
@@ -240,6 +275,22 @@ func ExecuteTask(task Task) Task {
 			taskResult := taskDisableTunnel()
 			task.Result = sql.NullString{String: taskResult, Valid: true}
 
+		case "start_shell":
+			log.Println("Starting SSHX.io Shell")
+			var args taskStartShellArgs
+			err := json.Unmarshal([]byte(task.Args.String), &args)
+			if err != nil {
+				log.Printf("Error reading arguments or start_shell task: %s", err)
+			} else {
+				taskResult := taskStartShell(args)
+				task.Result = sql.NullString{String: taskResult, Valid: true}												
+			}
+
+		case "stop_shell":
+			log.Println("Stopping SSHX.io Shell...")
+			taskResult := taskStopShell()
+			task.Result = sql.NullString{String: taskResult, Valid: true}
+
 		case "install_edgeapp":
 
 			log.Println("Installing EdgeApp...")
@@ -252,6 +303,18 @@ func ExecuteTask(task Task) Task {
 				task.Result = sql.NullString{String: taskResult, Valid: true}
 			}
 
+		case "install_bulk_edgeapps":
+
+			log.Println("Installing Bulk EdgeApps...")
+			var args taskInstallBulkEdgeAppsArgs
+			err := json.Unmarshal([]byte(task.Args.String), &args)
+			if err != nil {
+				log.Printf("Error reading arguments of install_bulk_edgeapps task: %s", err)
+			} else {
+				taskResult := taskInstallBulkEdgeApps(args)
+				task.Result = sql.NullString{String: taskResult, Valid: true}
+			}
+
 		case "remove_edgeapp":
 
 			log.Println("Removing EdgeApp...")
@@ -367,6 +430,25 @@ func ExecuteTask(task Task) Task {
 			taskResult := taskDisablePublicDashboard()
 			task.Result = sql.NullString{String: taskResult, Valid: true}
 
+		case "check_updates":
+			log.Println("Checking for updates...")
+			taskResult := taskCheckSystemUpdates()
+			task.Result = sql.NullString{String: taskResult, Valid: true}
+
+		case "apply_updates":
+
+			log.Println("Updating Edgebox System...")
+			is_updating := utils.ReadOption("UPDATING_SYSTEM")
+			if is_updating == "true"  {
+				log.Println("Edgebox update was running... Probably system restarted. Finishing update...")
+				utils.WriteOption("UPDATING_SYSTEM", "false")
+				task.Result = sql.NullString{String: "{result: true}", Valid: true}
+			} else {
+				log.Println("Updating Edgebox System...")
+				taskResult := taskUpdateSystem()
+				task.Result = sql.NullString{String: taskResult, Valid: true}
+			}
+
 		}
 
 	}
@@ -387,6 +469,7 @@ func ExecuteTask(task Task) Task {
 		if err != nil {
 			log.Fatal(err.Error())
 		}
+
 	} else {
 		fmt.Println("Error executing task with result: " + task.Result.String)
 		_, err = statement.Exec(STATUS_ERROR, "Error", formatedDatetime, strconv.Itoa(task.ID)) // Execute SQL Statement with Error info
@@ -440,6 +523,9 @@ func ExecuteSchedules(tick int) {
 		taskStartWs()
 		log.Println(taskGetEdgeApps())
 		taskUpdateSystemLoggerServices()
+		taskRecoverFromUpdate()
+		taskCheckSystemUpdates()
+		
 	}
 
 	if tick%5 == 0 {
@@ -486,6 +572,7 @@ func ExecuteSchedules(tick int) {
 
 	if tick%3600 == 0 {
 		// Executing every 3600 ticks (1 hour)
+		taskCheckSystemUpdates()
 	}
 
 	if tick%86400 == 0 {
@@ -904,12 +991,19 @@ func taskSetupTunnel(args taskSetupTunnelArgs) string {
 }
 
 func taskStartTunnel() string {
-	fmt.Println("Executing taskStartTunnel")
+    fmt.Println("Executing taskStartTunnel")
-	system.StartService("cloudflared")
+    
-	domainName := utils.ReadOption("DOMAIN_NAME")
+    // Read tunnel status to check if cloudflare is configured
-	status := "{\"status\": \"connected\", \"domain\": \"" + domainName + "\"}"
+    tunnelStatus := utils.ReadOption("TUNNEL_STATUS")
-	utils.WriteOption("TUNNEL_STATUS", status)
+	if tunnelStatus != "" {
-	return "{\"status\": \"ok\"}"
+		// Only start cloudflared if we have a tunnel configured
+        system.StartService("cloudflared")
+        domainName := utils.ReadOption("DOMAIN_NAME")
+        status := "{\"status\": \"connected\", \"domain\": \"" + domainName + "\"}"
+        utils.WriteOption("TUNNEL_STATUS", status)
+	}
+    
+    return "{\"status\": \"ok\"}"
 }
 
 func taskStopTunnel() string {
@@ -931,6 +1025,80 @@ func taskDisableTunnel() string {
 	return "{\"status\": \"ok\"}"
 }
 
+func taskStartShell(args taskStartShellArgs) string {
+	fmt.Println("Executing taskStartShell")
+	wsPath := utils.GetPath(utils.WsPath)
+
+	// kill the process if its running
+	utils.Exec(wsPath, "killall", []string{"sshx"})
+
+	cmd := exec.Command("/usr/local/bin/sshx", "--quiet", "--shell", "bash")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		panic(err)
+	}
+	scanner := bufio.NewScanner(stdout)
+	err = cmd.Start()
+	if err != nil {
+		panic(err)
+	}
+	url := ""
+
+	timeout := args.Timeout
+
+	for scanner.Scan() {
+		fmt.Println(scanner.Text())
+		text := scanner.Text()
+		if strings.Contains(text, "https://") {
+			url = text
+			fmt.Println("Shell start is responding with URL: " + url)
+			utils.WriteOption("SHELL_URL", url)
+			utils.WriteOption("SHELL_STATUS", "running")
+			break
+		}
+	}
+	if scanner.Err() != nil {
+		cmd.Process.Kill()
+		cmd.Wait()
+		panic(scanner.Err())
+	}
+
+	go func() {
+		fmt.Println("Running shell async")
+		
+		// cmd.Wait()
+
+		// Keep retrying to calculate timeout to know when to kill the process
+		for {
+			timeout = timeout - 1
+			if timeout <= 0 {
+				fmt.Println("Timeout reached, killing process...")
+				utils.Exec(wsPath, "killall sshx", []string{})
+				utils.WriteOption("SHELL_STATUS", "not_running")
+				break
+			}
+			if timeout%10 == 0 {
+				fmt.Println("Active Shell Timeout is " + fmt.Sprint(timeout) + " seconds")
+			}
+			time.Sleep(1 * time.Second)
+		}
+	}()
+
+	return "{\"status\": \"ok\"}"
+}
+
+func taskStopShell() string {
+	fmt.Println("Executing taskStopShell")
+	wsPath := utils.GetPath(utils.WsPath)
+
+	// kill the process if its running
+	utils.Exec(wsPath, "killall", []string{"sshx"})
+	utils.WriteOption("SHELL_STATUS", "not_running")
+
+	return "{\"status\": \"ok\"}"
+
+}
+
 func taskInstallEdgeApp(args taskInstallEdgeAppArgs) string {
 	fmt.Println("Executing taskInstallEdgeApp for " + args.ID)
 
@@ -941,6 +1109,16 @@ func taskInstallEdgeApp(args taskInstallEdgeAppArgs) string {
 	return string(resultJSON)
 }
 
+func taskInstallBulkEdgeApps(args taskInstallBulkEdgeAppsArgs) string {
+	fmt.Println("Executing taskInstallBulkEdgeApps for " + strings.Join(args.IDS, ", "))
+
+	// args.Apps is a list of edgeapp ids
+	edgeapps.SetEdgeAppBulkInstalled(args.IDS)
+
+	taskGetEdgeApps()
+	return "{\"status\": \"ok\"}"
+}
+
 func taskRemoveEdgeApp(args taskRemoveEdgeAppArgs) string {
 	fmt.Println("Executing taskRemoveEdgeApp for " + args.ID)
 
@@ -1138,6 +1316,39 @@ func taskDisablePublicDashboard() string {
 	return "{result: false}"
 }
 
+func taskCheckSystemUpdates() string {
+	fmt.Println("Executing taskCheckSystemUpdates")
+	system.CheckUpdates()
+	return "{result: true}"
+}
+
+func taskUpdateSystem() string {
+	fmt.Println("Executing taskUpdateSystem")
+	system.ApplyUpdates()
+	utils.WriteOption("LAST_UPDATE", strconv.FormatInt(time.Now().Unix(), 10))
+	return "{result: true}"
+}
+
+func taskRecoverFromUpdate() string {
+	fmt.Println("Executing taskRecoverFromUpdate")
+	executing_tasks := GetExecutingTasks()
+	// Filter out the task with task value "update_system"
+	filteredTasks := []Task{}
+	for _, task := range executing_tasks {
+		if task.Task != "update_system" {
+			filteredTasks = append(filteredTasks, task)
+		}
+	}
+	
+	// If tasks is not empty, Get the last task
+	if len(filteredTasks) > 0 {
+		lastTask := filteredTasks[len(filteredTasks)-1]
+		ExecuteTask(lastTask)
+	}
+
+	return "{result: true}"
+}
+
 func taskSetReleaseVersion() string {
 
 	fmt.Println("Executing taskSetReleaseVersion")