diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..997ca2f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vagrant \ No newline at end of file diff --git a/README.md b/README.md index 533e169..427c587 100644 --- a/README.md +++ b/README.md @@ -1 +1,19 @@ # arduino-builder-service + +Arduino Builder Service modifies codebender/compiler to compile skits with arduino-builder. As of now Arduino Builder Service compiles for version 166 for Arduino 1.6.6. + +## Setup Arduino Builder Service with Vagrant +Install [Vagrant](https://www.vagrantup.com/downloads.html) on your machine. + +Next, copy the code into any directory. `git clone https://github.com/codebendercc/arduino-builder-service.git` +Go into the directory with the code and `vagrant up` +SSH into the virtual machine using +``` +host: localhost +port: 2222 +username: vagrant +password: vagrant + +``` +Finally, set up the virtual machine by running `/vagrant/setup.sh` +To test if it work run `curl https://localhost/status` should return {"success":true,"status":"OK"}. diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..93a0c4f --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,71 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "ubuntu/trusty64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = true + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network "forwarded_port", guest: 80, host: 80 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + #config.vm.synced_folder "./", "/home/vagrant" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + vb.memory = "2048" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell", inline: <<-SHELL + sudo apt-get update + #sudo apt-get install -y apache2 + SHELL +end diff --git a/modified-files/CompilerBundle/Controller/DefaultController.php b/modified-files/CompilerBundle/Controller/DefaultController.php new file mode 100644 index 0000000..e30cda1 --- /dev/null +++ b/modified-files/CompilerBundle/Controller/DefaultController.php @@ -0,0 +1,180 @@ + true, "status" => "OK"))); + } + + public function testAction($authorizationKey) + { + $params = $this->generateParameters(); + + if ($authorizationKey !== $params["authorizationKey"]) + { + return new Response(json_encode(array("success" => false, "step" => 0, "message" => "Invalid authorization key."))); + } + + set_time_limit(0); // make the script execution time unlimited (otherwise the request may time out) + + // change the current Symfony root dir + chdir($this->get('kernel')->getRootDir()."/../"); + + //TODO: replace this with a less horrible way to handle phpunit + exec("phpunit -c app --stderr 2>&1", $output, $return_val); + + return new Response(json_encode(array("success" => (bool) !$return_val, "message" => implode("\n", $output)))); + } + + public function indexAction($authorizationKey, $version) + { + $params = $this->generateParameters(); + + if ($authorizationKey !== $params["authorizationKey"]) + { + return new Response(json_encode(array("success" => false, "step" => 0, "message" => "Invalid authorization key."))); + } + + if ($version == "v1") + { + $request = $this->getRequest()->getContent(); + + //Get the compiler service + /** @var CompilerHandler $compiler */ + $compiler = $this->get('builder_compiler_handler'); + + $reply = $compiler->main($request, $params); + + return new Response(json_encode($reply)); + } + else + { + return new Response(json_encode(array("success" => false, "step" => 0, "message" => "Invalid API version."))); + } + } + + public function deleteAllObjectsAction($authorizationKey, $version) + { + if ($this->container->getParameter('authorizationKey') != $authorizationKey) { + return new Response(json_encode( + array('success' => false, 'step' => 0, 'message' => 'Invalid authorization key.') + )); + } + + if ($version != 'v1') { + return new Response(json_encode( + array('success' => false, 'step' => 0, 'message' => 'Invalid API version.') + )); + } + + //Get the compiler service + /** @var DeletionHandler $deleter */ + $deleter = $this->get('deletion_handler'); + + $response = $deleter->deleteAllObjects(); + + if ($response['success'] !== true) { + return new Response(json_encode( + array('success' => false, 'step' => 0, 'message' => 'Failed to access object files directory.') + )); + } + + return new Response(json_encode( + array_merge( + array( + 'success' => true, + 'message' => 'Object files deletion complete. Found ' . $response['fileCount'] . ' files.' + ), + $response['deletionStats'], + array("Files not deleted" => $response['notDeletedFiles']) + ))); + } + + public function deleteSpecificObjectsAction($authorizationKey, $version, $option, $cachedObjectToDelete) + { + if ($this->container->getParameter('authorizationKey') != $authorizationKey) { + return new Response(json_encode( + array('success' => false, 'step' => 0, 'message' => 'Invalid authorization key.') + )); + } + + if ($version != 'v1') { + return new Response(json_encode( + array('success' => false, 'step' => 0, 'message' => 'Invalid API version.') + )); + } + + //Get the compiler service + /** @var DeletionHandler $deleter */ + $deleter = $this->get('deletion_handler'); + + $response = $deleter->deleteSpecificObjects($option, $cachedObjectToDelete); + + if ($response['success'] !== true) { + return new Response(json_encode( + array('success' => false, 'step' => 0, 'message' => 'Failed to access object files directory.') + )); + } + + if (!empty($response["notDeletedFiles"])) { + $message = 'Failed to delete one or more of the specified core object files.'; + if ($option == 'library') { + $message = 'Failed to delete one or more of the specified library object files.'; + } + + return new Response(json_encode( + array_merge(array('success' => false, 'step' => 0, 'message' => $message), $response) + )); + } + + $message = 'Core object files deleted successfully.'; + if ($option == 'library'){ + $message = 'Library deleted successfully.'; + } + + return new Response(json_encode(array_merge(array('success' => true, 'message' => $message), $response))); + } + + /** + * \brief Creates a list of the configuration parameters to be used in the compilation process. + * + * \return An array of the parameters. + * + * This function accesses the Symfony global configuration parameters, and creates an array that our handlers (which + * don't have access to them) can use them. + + */ + private function generateParameters() + { + $parameters = array("binutils", "python", "clang", "logdir", "temp_dir", "archive_dir", "autocompletion_dir", "autocompleter", "cflags", "cppflags", "asflags", "arflags", "ldflags", "ldflags_tail", "clang_flags", "objcopy_flags", "size_flags", "output", "arduino_cores_dir", "external_core_files", "authorizationKey"); + + $compiler_config = array(); + + foreach ($parameters as $parameter) + { + $compiler_config[$parameter] = $this->container->getParameter($parameter); + } + + return $compiler_config; + } + +} diff --git a/modified-files/CompilerBundle/Handler/BuilderCompilerHandler.php b/modified-files/CompilerBundle/Handler/BuilderCompilerHandler.php new file mode 100644 index 0000000..ea8857f --- /dev/null +++ b/modified-files/CompilerBundle/Handler/BuilderCompilerHandler.php @@ -0,0 +1,262 @@ +preproc = $preprocHandl; + $this->postproc = $postprocHandl; + $this->utility = $utilHandl; + $this->compiler_logger = $logger; + $this->object_directory = $objdir; + } + + + function main($request, $compiler_config) + { + + error_reporting(E_ALL & ~E_STRICT); + + + $start_time = microtime(true); + + // Step 0: Reject the request if the input data is not valid. + $tmp = $this->requestValid($request); + if($tmp["success"] == false) + return $tmp; + + $this->setVariables($request, $format, $libraries, $version, $mcu, $f_cpu, $core, $variant, $vid, $pid, $compiler_config); + + $TEMP_DIR = $compiler_config["temp_dir"]; + + // Step 1: Extract the files and filenames included in the request. + $filenames = array(); + + foreach ($request["files"] as $file) + array_push($filenames, $file["filename"]); + + + + $files = array(); + $tmpVar = $this->extractFiles($request["files"], $TEMP_DIR, $compiler_dir, $files["sketch_files"], "files"); + if ($tmp["success"] == false) + return $tmp; + + if (!array_key_exists("archive", $request) || ($request["archive"] !== false && $request["archive"] !== true)) + $ARCHIVE_OPTION = false; + else + $ARCHIVE_OPTION = $request["archive"]; + + if ($ARCHIVE_OPTION === true) + { + $arch_ret = $this->createArchive($compiler_dir, $TEMP_DIR, $ARCHIVE_DIR, $ARCHIVE_PATH); + if ($arch_ret["success"] === false) + return $arch_ret; + } + + + $tmp = $this->doCompile($files["sketch_files"], $compiler_dir, $format, $version, $mcu, $f_cpu, $core, $variant, $pid, $vid); + if ($tmp["success"] == false) + return $tmp; + + + + if ($format == "syntax") + return array( + "success" => true, + "time" => microtime(true) - $start_time); + + + $tmp = $this->convertOutput($compiler_dir, $format, $SIZE, $filenames , $start_time); + return $tmp; + + + } + + protected function convertOutput($dir, $format, $SIZE, $filenames, $start_time) + { + $OUTPUT = $filenames[0]; + // To Do, Handle multiple files than just one. + + if ($format == "elf") + { + $ret_objcopy = false; + $content = base64_encode(file_get_contents("$dir/$OUTPUT.elf")); + $size = filesize("$dir/$OUTPUT.elf"); + } + elseif ($format == "binary") + { + $content = base64_encode(file_get_contents("$dir/$OUTPUT.hex")); + $size = filesize("$dir/$OUTPUT.hex"); + } + elseif ($format == "hex") + { + $content = file_get_contents("$dir/$OUTPUT.hex"); + $size = filesize("$dir/$OUTPUT.hex"); + } + elseif ($format == "object") + { + + $content = base64_encode(file_get_contents("$dir/sketch/$OUTPUT.cpp.o")); + $size = filesize("$dir/sketch/$OUTPUT.cpp.o"); + } + + // If everything went well, return the reply to the caller. + if ($ret_objcopy || $content === false) + return array( + "success" => false, + "step" => 8, + "message" => "There was a problem while generating the your binary file for $dir/sketch/$OUTPUT.cpp.o"); + else + return array( + "success" => true, + "time" => microtime(true) - $start_time, + "size" => $size, + "output" => $content); + + } + private function requestValid(&$request) + { + $request = $this->preproc->validateInput($request); + if (!$request) + return array( + "success" => false, + "step" => 0, + "message" => "Invalid input."); + else return array("success" => true); + } + + private function extractFiles($request, $temp_dir, &$dir, &$files, $suffix, $lib_extraction = false) + { + // Create a temporary directory to place all the files needed to process + // the compile request. This directory is created in $TMPDIR or /tmp by + // default and is automatically removed upon execution completion. + + $cnt = 0; + + if (!$dir) + do + { + $dir = @System::mktemp(" -t $temp_dir/ -d compiler"); + $cnt++; + } while (!$dir && $cnt <= 2); + + if (!$dir) + return array( + "success" => false, + "step" => 1, + "message" => "Failed to create temporary directory."); + + $response = $this->utility->extractFiles("$dir/$suffix", $request, $lib_extraction); + + if ($response["success"] === false) + return $response; + $files = $response["files"]; + + return array("success" => true); + } + + + protected function doCompile(&$files, $dir, $format, $version, $mcu, $f_cpu, $core, $variant, $pid, $vid) + { + $hardware = "-hardware /opt/codebender/arduino-core-files/v1.6.6/hardware -hardware /opt/codebender/arduino-core-files/v1.6.6/hardware/arduino"; + $tools = "-tools /opt/arduino-builder/tools-builder -tools /opt/codebender/arduino-core-files/v1.6.6/hardware/tools/avr"; + $libraries = "-libraries /opt/codebender/arduino-core-files/v1.6.6/libraries"; + $build_path = "-build-path $dir"; + + if(!file_exists($dir)) + mkdir($dir); + + $prefs = "-prefs mcu=$mcu -prefs f_cpu=$f_cpu -prefs core=$core -prefs variant=$variant "; + + // If pid and vid build preferences were specified + if($pid != "") + $prefs = $prefs."-prefs pid=$pid "; + + if($vid != "") + $prefs = $prefs."-prefs vid=$vid "; + + $hardware = stripslashes($hardware); + $tools = stripslashes($tools); + $libraries = stripslashes($libraries); + $build_path = stripslashes($build_path); + $prefs = stripslashes($prefs); + + foreach (array("c", "cpp", "S","ino") as $ext) + { + foreach ($files[$ext] as $file) + { + // From hereon, $file is shell escaped and thus should only be used in calls + // to exec(). + $file = escapeshellarg($file); + + exec("/opt/arduino-builder/arduino-builder -fqbn arduino:avr:uno $hardware $tools $libraries $prefs $build_path $file.$ext 2>&1", $output, $ret_compile); + + + + if (isset($ret_compile) && $ret_compile) + { + return array( + "success" => false, + "step" => 4, + "message" => $output, + "debug" => $ret_compile); + } + unset($output); + + $files["o"][] = array_shift($files[$ext]); + + } + } + + return array("success" => true); + } + + + private function setVariables($request, &$format, &$libraries, &$version, &$mcu, &$f_cpu, &$core, &$variant, &$vid, &$pid, &$compiler_config) + { + // Extract the request options for easier access. + $format = $request["format"]; + $libraries = $request["libraries"]; + $version = $request["version"]; + $mcu = $request["build"]["mcu"]; + $f_cpu = $request["build"]["f_cpu"]; + $core = $request["build"]["core"]; + // Some cores do not specify any variants. In this case, set variant to be an empty string + if (!array_key_exists("variant", $request["build"])) + $variant = ""; + else + $variant = $request["build"]["variant"]; + + if ($format == "autocomplete") + { + $compiler_config["autocmpfile"] = $request["position"]["file"]; + $compiler_config["autocmprow"] = $request["position"]["row"]; + $compiler_config["autocmpcol"] = $request["position"]["column"]; + $compiler_config["autocmpmaxresults"] = 500; + $compiler_config["autocmpprefix"] = $request["prefix"]; + } + + // Set the appropriate variables for vid and pid (Leonardo). + + $vid = (isset($request["build"]["vid"])) ? $request["build"]["vid"] : "null"; + $pid = (isset($request["build"]["pid"])) ? $request["build"]["pid"] : "null"; + } +} diff --git a/modified-files/CompilerBundle/Resources/config/routing.yml b/modified-files/CompilerBundle/Resources/config/routing.yml new file mode 100644 index 0000000..bb0f875 --- /dev/null +++ b/modified-files/CompilerBundle/Resources/config/routing.yml @@ -0,0 +1,27 @@ +codebender_compiler_status_check: + pattern: /status + defaults: { _controller: CodebenderCompilerBundle:Default:status } + +codebender_compiler_test: + pattern: /{authorizationKey}/test/ + defaults: { _controller: CodebenderCompilerBundle:Default:test } + +codebender_compiler_homepage: + pattern: /{authorizationKey}/{version} + defaults: { _controller: CodebenderCompilerBundle:Default:index } + +codebender_compiler_delete_all: + pattern: /{authorizationKey}/{version}/delete/all/ + defaults: { _controller: CodebenderCompilerBundle:Default:deleteAllObjects } + +codebender_compiler_delete_specific: + pattern: /{authorizationKey}/{version}/delete/{option}/{cachedObjectToDelete} + defaults: { _controller: CodebenderCompilerBundle:Default:deleteSpecificObjects } + +# redirecting the root +root: + path: / + defaults: + _controller: FrameworkBundle:Redirect:urlRedirect + path: /status + permanent: true diff --git a/modified-files/CompilerBundle/Resources/config/services.yml b/modified-files/CompilerBundle/Resources/config/services.yml new file mode 100644 index 0000000..6a9ac30 --- /dev/null +++ b/modified-files/CompilerBundle/Resources/config/services.yml @@ -0,0 +1,36 @@ +parameters: +# codebender_compiler.example.class: Codebender\CompilerBundle\Example + buildercompiler_handler.class: Codebender\CompilerBundle\Handler\BuilderCompilerHandler + compiler_handler.class: Codebender\CompilerBundle\Handler\CompilerHandler + utility_handler.class: Codebender\CompilerBundle\Handler\UtilityHandler + preprocessing_handler.class: Codebender\CompilerBundle\Handler\PreprocessingHandler + postprocessing_handler.class: Codebender\CompilerBundle\Handler\PostprocessingHandler + deletion_handler.class: Codebender\CompilerBundle\Handler\DeletionHandler + +services: +# codebender_compiler.example:- +# class: %codebender_compiler.example.class% +# arguments: [@service_id, "plain_value", %parameter%] + buildercompiler_handler: + class: "%buildercompiler_handler.class%" + arguments: ["@preprocessing_handler", "@postprocessing_handler", "@utility_handler", "@compiler_logger", "%temp_dir%/%objdir%"] + compiler_handler: + class: "%compiler_handler.class%" + arguments: ["@preprocessing_handler", "@postprocessing_handler", "@utility_handler", "@compiler_logger", "%temp_dir%/%objdir%"] + utility_handler: + class: "%utility_handler.class%" + preprocessing_handler: + class: "%preprocessing_handler.class%" + postprocessing_handler: + class: "%postprocessing_handler.class%" + deletion_handler: + class: "%deletion_handler.class%" + arguments: ["%temp_dir%/%objdir%"] + compiler_logger: + class: Symfony\Bridge\Monolog\Logger + arguments: [cmplr_log] + calls: + - [pushHandler, [@compiler_log_handler]] + compiler_log_handler: + class: Monolog\Handler\StreamHandler + arguments: [%kernel.logs_dir%/compiler.log] diff --git a/setup.sh b/setup.sh new file mode 100644 index 0000000..b8a94fa --- /dev/null +++ b/setup.sh @@ -0,0 +1,16 @@ +cd /home/vagrant +sudo apt-get install git +git clone https://github.com/codebendercc/compiler.git +cp -r /vagrant/modified-files/CompilerBundle/ compiler/Symfony/src/Codebender/ +cd /home/vagrant/compiler/ +./scripts/install.sh +cd /home/vagrant +wget http://arduino.cc/download.php?f=/arduino-1.6.6-linux64.tar.xz -O arduino-1.6.6.tar.xz +tar -xJf arduino-1.6.6.tar.xz +sudo mkdir /opt/codebender/arduino-core-files +sudo mkdir /opt/codebender/arduino-core-files/v1.6.6 +sudo cp -r arduino-1.6.6/hardware /opt/codebender/arduino-core-files/v1.6.6/ +sudo cp -r arduino-1.6.6/libraries /opt/codebender/arduino-core-files/v1.6.6/ +sudo mkdir /opt/arduino-builder +sudo cp -r arduino-1.6.6/tools-builder /opt/arduino-builder/ +sudo cp -r arduino-1.6.6/arduino-builder /opt/arduino-builder/ \ No newline at end of file