diff --git a/Dockerfile b/Dockerfile index 4b46f2ee..4bc498d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE="ubuntu:20.04" +ARG BASE_IMAGE="ubuntu:22.04" ### BASE IMAGE FROM $BASE_IMAGE as base @@ -6,11 +6,15 @@ ARG BASE_IMAGE ENV DEBIAN_FRONTEND=noninteractive ENV LLVM_DIR=/usr/lib/llvm-11 +ENV PATH="/scripts:${PATH}" +ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python # Copy dependencies lists into container. We copy them all and then do a mv because # we need to transform base_image into a windows compatible filename which we can't # do in a COPY command. COPY ./dependencies/* /tmp +COPY ./requirements.txt /tmp + RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt && \ mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_base.txt /tmp/base_dep.txt @@ -21,9 +25,9 @@ RUN [ -e /tmp/base_dep.txt ] && \ apt-get clean # Finally: Install panda debian package, you need a version that has the Dwarf2 Plugin -RUN wget https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_20.04.deb -RUN command apt install -qq -y ./pandare_20.04.deb -RUN pip install pandare +RUN curl -LJO https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_22.04.deb +RUN apt install -qq -y ./pandare_22.04.deb +RUN pip install -r /tmp/requirements.txt ### BUILD IMAGE - STAGE 2 FROM base AS builder @@ -38,13 +42,16 @@ RUN [ -e /tmp/build_dep.txt ] && \ FROM builder as developer COPY ./tools/ /tools -COPY setup_container.py / +COPY ./scripts /scripts + +# Essentially same as setup_container.sh +RUN cd /tools/btrace && ./compile.sh -RUN python3 setup_container.py +RUN rm -rf /tools/build +RUN mkdir -p /tools/build +RUN mkdir -p /tools/install -# RUN cd /tools && \ -# cmake -Bbuild -H. -DLLVM_DIR=$LLVM_DIR/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=/tools/install +RUN cmake -B"/tools/build" -H"/tools" -DCMAKE_INSTALL_PREFIX="/tools/install" +RUN make --no-print-directory -j4 install -C "/tools/build/lavaTool" -# RUN cd /tools/build && \ -# make && \ -# make install V=1 \ No newline at end of file +RUN make --no-print-directory -j4 install -C "/tools/build/fbi" diff --git a/SETUP.md b/SETUP.md deleted file mode 100644 index 90e3cd1d..00000000 --- a/SETUP.md +++ /dev/null @@ -1,49 +0,0 @@ -# LAVA Installation Guide - -## Docker installation - sudo apt-get install docker.io - -## If that doesn’t work: - sudo apt-get update - sudo apt-get upgrade //optional - - sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 \ - --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - - touch /etc/apt/source.list.d/docker.list - - echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" | sudo tee - /etc/apt/sources.list.d/docker.list - - sudo apt-get update - - sudo apt-get install docker-engine - sudo service docker start -NB: Change the distribution version name accordingly - - -## Git and Python installation - sudo apt-get install git - sudo apt-get install python - sudo apt-get install python-pip - sudo pip install --upgrade pip - sudo pip install colorama - -## Grant docker usage for non-root - sudo usermod -a -G docker $USER - su - $USER - docker ps //test - -## Clone the repository - git clone git@bitbucket.org:moyix/lava.git - - or - - git clone https://$YOUR_USERNAME@bitbucket.org/moyix/lava.git - -## Install LAVA - cd lava - python setup.py - -## Try LAVA out - python init_project.py diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 326d9be3..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM pandare/panda - -RUN apt-get install -y sudo build-essential mlocate \ - vim libc++-dev g++-multilib g++ ninja-build \ - jq bc cmake autoconf libtool zlib1g-dev \ - llvm-11-dev libclang-11-dev \ - libpq-dev - -ENV release 11 -ENV llvm_version llvm-${release} - -WORKDIR / -RUN wget http://codesynthesis.com/download/odb/2.4/odb_2.4.0-1_amd64.deb -RUN dpkg -i odb_2.4.0-1_amd64.deb -#RUN wget http://codesynthesis.com/download/odb/2.4/odb-2.4.0.tar.gz -RUN wget http://codesynthesis.com/download/odb/2.4/libodb-2.4.0.tar.gz -RUN tar xf libodb-2.4.0.tar.gz -WORKDIR /libodb-2.4.0 -RUN ./configure --enable-shared && make -j $(nproc) && make install -WORKDIR / -RUN wget http://codesynthesis.com/download/odb/2.4/libodb-pgsql-2.4.0.tar.gz -RUN tar xf libodb-pgsql-2.4.0.tar.gz -WORKDIR /libodb-pgsql-2.4.0 -RUN ./configure --enable-shared && make -j $(nproc) && make install - -RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/usr-local-lib.conf -RUN ldconfig - -RUN pip install --upgrade -v pip -i https://pypi.python.org/simple/ -RUN pip install subprocess32 lockfile sqlalchemy==1.0.14 -i https://pypi.python.org/simple - -RUN pip install pyyaml pycparser psycopg2 -RUN updatedb - -RUN echo "LLVM_DIR=/usr/lib/$llvm_version/lib/cmake/llvm" >> /etc/environment -RUN echo "LIBRARY_PATH=/usr/local/lib" >> /etc/environment - -# Setting up remote Postgres(host) && PANDA Aux Packages -RUN apt-get install -y genisoimage postgresql-client-12 -# pg_hba.conf: -# host all all 172.17.0.1/16 md5 -# postgresql.conf: -# listen_addresses = 'localhost, 172.17.0.1' diff --git a/docker/requirements.txt b/docker/requirements.txt deleted file mode 100644 index f3fe750c..00000000 --- a/docker/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -SQLAlchemy==1.0.14 -zmq==0.0.0 -pyzmq==15.2.0 -psycopg2==2.6.1 -lockfile==0.10.2 -tabulate==0.7.5 -pexpect==3.2 -psutil==5.6.6 -ipython==2.3.0 -colorama==0.3.2 -numpy==1.8.2 -argparse==1.2.1 -pandare diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 00000000..c86c43c8 --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,24 @@ +# Setup LAVA + +## Ubuntu 64-bit 16.04.4 +The following install steps worked on 6/29/2019 with LAVA commit [c55bf1826ef9855a621f2652b30f16ac75b19cb6](https://github.com/panda-re/lava/commit/c55bf1826ef9855a621f2652b30f16ac75b19cb6). + +- Download and install [Ubuntu 64-bit 16.04.6](http://releases.ubuntu.com/16.04/ubuntu-16.04.6-desktop-amd64.iso) (SHA1 A09607901183AB25C675626024AA402663FA2558, MD5: 5416371CC0E990871746DDAAC89F1A5E). +- `sudo add-apt-repository ppa:phulin/panda` +- `sudo cp /etc/apt/sources.list /etc/apt/sources.list~` +- `sudo sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list` +- `sudo apt-get update` +- `sudo apt-get install python-pip git protobuf-compiler protobuf-c-compiler libprotobuf-c0-dev libprotoc-dev python-protobuf libelf-dev libcapstone-dev libdwarf-dev python-pycparser llvm-3.3 clang-3.3 libc++-dev libwiretap-dev libwireshark-dev odb` +- `sudo apt-get build-dep qemu` +- `pip install colorama` +- `cd ~/Desktop` +- `git clone https://github.com/panda-re/lava.git` +- `cd ~/Desktop/lava` +- `python2 setup.py` + +## Updated + +- build Docker container with `docker/Dockerfile` +- use `scripts/docker-shell.sh` to enter docker container +- inside Docker container, run `python setup_container.py` to build lavaTools +- exit Docker container, and run `python setup_postgres.py` and `init-host.py` diff --git a/init-host.py b/init-host.py index 4d87ac51..176b53bb 100755 --- a/init-host.py +++ b/init-host.py @@ -18,13 +18,11 @@ from colorama import Fore from colorama import Style -QCOW_URL = "http://panda.moyix.net/~moyix/wheezy_panda2.qcow2" # if moyix server is down, this image will also work -# QCOW_URL = "https://panda.re/qcows/linux/debian/7.3/x86/debian_7.3_x86.qcow" +QCOW_URL = "https://panda.re/qcows/linux/debian/7.3/x86/debian_7.3_x86.qcow" QCOW_FILE_NAME = "wheezy_panda2.qcow2" TAR_URL = "ftp://ftp.astron.com/pub/file/file-5.22.tar.gz" LAVA_DIR = dirname(abspath(sys.argv[0])) -os.chdir(LAVA_DIR) def progress(msg): @@ -57,20 +55,7 @@ def run(cmd): def main(): - # try to import lava.mak as a config file if not exit - try: - def_lines = (line.strip() for line in open("lava.mak", "r") - if not line.strip().startswith("#") - and line.strip() != "") - def_lines = (line.split(":=") for line in def_lines) - def_lines = ((line[0].strip(), line[1].strip()) for line in def_lines) - LAVA_CONFS = dict(def_lines) - PANDA_BUILD_DIR = LAVA_CONFS["PANDA_BUILD_DIR"] - PANDA_BUILD_DIR = expandvars(PANDA_BUILD_DIR) - print("PANDA_BUILD_DIR Used {}".format(PANDA_BUILD_DIR)) - except Exception: - error("Make sure to have properly configured lava.mak \ - generated by setup.py") + # parser = argparse.ArgumentParser(description='Setup LAVA') # parser.add_argument('-s', '--skip_docker_build', action='store_true', # default = False, @@ -93,7 +78,7 @@ def main(): else: progress("Found existing target_bins/{}".format(basename(TAR_URL))) - if not isfile(join(LAVA_DIR, basename(QCOW_URL))): + if not isfile(join(LAVA_DIR, basename(QCOW_FILE_NAME))): progress("Downloading {}".format(basename(QCOW_URL))) run(["wget", "--no-check-certificate", QCOW_URL, "-O", QCOW_FILE_NAME]) else: @@ -103,8 +88,7 @@ def main(): progress("Building host.json") # Build host.json json_configs = {} - json_configs["qemu"] = join(join(PANDA_BUILD_DIR, "i386-softmmu"), - "qemu-system-i386") + json_configs["qemu"] = "panda-system-i386" json_configs["qcow_dir"] = LAVA_DIR json_configs["output_dir"] = join(LAVA_DIR, "target_injections") json_configs["config_dir"] = join(LAVA_DIR, "target_configs") @@ -119,15 +103,7 @@ def main(): else: progress("Found existing host.json") - # progress("(re)building the fbi") - # os.chdir(join(LAVA_DIR, "tools", "build", "fbi")) - # run(["make", "install", "-j4"]) - - # progress("(re)building lavaTool") - # os.chdir(join(LAVA_DIR, "tools", "build", "lavaTool")) - # run(["./compile-on-docker.sh"]) - - progress("Sucessful! Now run:\n $ scripts/lava.sh -ak file") + progress("Successful! Now run:\n $ scripts/lava.sh -ak file") return 0 diff --git a/install.sh b/install.sh index 7c7d7e8c..ce99bb82 100755 --- a/install.sh +++ b/install.sh @@ -13,15 +13,6 @@ progress() { echo -e "\e[32m[lava_install]\e[0m \e[1m$1\e[0m" } -# Step 1: Install panda debian package -wget https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_22.04.deb -$SUDO apt install ./pandare_22.04.deb - -# Remove the comment to update all debian stuff from sources.list -$SUDO cp /etc/apt/sources.list /etc/apt/sources.list~ -$SUDO sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list -$SUDO apt-get update - progress "Updates complete" # This was originally in the docs/setup.md, I removed things starting with 'python-' as that should be installed via pip @@ -53,12 +44,10 @@ $SUDO apt install ./pandare_22.04.deb progress "Installed build dependencies" -$SUDO pip3 install --upgrade pip - -# This seems to be the better replacement to have all python packages -$SUDO pip3 install -r docker/requirements.txt +pip3 install --upgrade pip +pip3 install -r requirements.txt progress "Installed Python requirements" -$SUDO python3 setup_container.py +$SUDO bash ./setup_container.sh progress "Installed LAVA" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d2675377 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +colorama +db +ipython +lockfile +numpy +pandare +PyYAML +SQLAlchemy +tabulate diff --git a/scripts/manual_lava.py b/scripts/manual_lava.py index 56a4b880..0da93626 100644 --- a/scripts/manual_lava.py +++ b/scripts/manual_lava.py @@ -1,12 +1,6 @@ -import re import sys -import random import subprocess as sp import sys - -# ok this is gross -sys.path.append("/home/tleek/git/panda-leet/panda/scripts") - from pandare.plog_reader import PLogReader # diff --git a/scripts/setup_postgres.sh b/scripts/setup_postgres.sh new file mode 100644 index 00000000..94f7b36d --- /dev/null +++ b/scripts/setup_postgres.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -ex + +PGPASS="${HOME}/.pgpass" + +if [ ! -f "${PGPASS}" ]; then + postgres_depends=$(dpkg-query -W -f='${depends}' 'postgresql') + postgres_pkg=$(echo "${postgres_depends}" | grep -oP 'postgresql-[0-9]+.?[0-9]+') + postgres_version=${postgres_pkg/postgresql-/} + pg_hba="/etc/postgresql/${postgres_version}/main/pg_hba.conf" + postgres_password='postgrespostgres' + + sudo sed -i.bak -E 's/^(local\s+all\s+postgres\s+)md5$/\1peer/' "${pg_hba}" + sudo service postgresql reload + + password_sql="ALTER USER postgres WITH PASSWORD '${postgres_password}';" + sudo -u postgres psql -c "${password_sql}" + + echo "*:*:*:postgres:${postgres_password}" > "${PGPASS}" + chmod 600 "${PGPASS}" + + sudo sed -i.bak -E 's/^(local\s+all\s+postgres\s+)peer$/\1md5/' "${pg_hba}" + sudo service postgresql reload +fi \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100755 index 0f6824fa..00000000 --- a/setup.py +++ /dev/null @@ -1,320 +0,0 @@ -#!/usr/bin/python - -import os -import re -import sys -import stat -import shlex -import getpass -import argparse -import subprocess - -from os.path import join -from os.path import isdir -from os.path import isfile -from os.path import abspath -from os.path import dirname -from os.path import expandvars - -from colorama import Fore -from colorama import Style - -from multiprocessing import cpu_count - - -def command_exited_nonzero(cmd): - try: - with open(os.devnull, 'w') as devnull: - subprocess.check_output(cmd, - shell=True, - stderr=devnull) - return False - except subprocess.CalledProcessError: - return True - - -def is_package_installed(pkg): - if pkg == "docker.io": - return os.path.isfile("/usr/bin/docker") - if (os.path.isfile(os.path.join("/usr/bin", pkg)) or - os.path.isfile(os.path.join("/bin", pkg))): - return True - if command_exited_nonzero("dpkg -s {}".format(pkg)): - # maybe it is a python package - try: - python_pkg = pkg.split("python-")[1] - return not command_exited_nonzero("python -c \"import {}\"" - .format(python_pkg)) - # pkg is not a string of "python-{}" - except IndexError: - return False - else: - return True - - -if not is_package_installed("python-colorama"): - subprocess.check_call(['sudo', 'apt-get', - 'install', '-y', 'python-colorama']) - -# this is set to denote user is already in docker group -ALREADY_IN_DOCKER_GROUP = False -LLVM_VERSION = "3.6.2" -DOCKER_NAME = "lava32" - -LAVA_DIR = dirname(abspath(sys.argv[0])) -os.chdir(LAVA_DIR) - -BUILD_DIR = join(os.environ["HOME"], "build") -try: - os.mkdir(BUILD_DIR) -except Exception: - pass - -# try to import lava.mak as a config file -# if not then resort to default locations for llvm and panda -try: - def_lines = (line.strip() for line in open("lava.mak", "r") - if not line.strip().startswith("#") - and line.strip() != "") - def_lines = (line.split(":=") for line in def_lines) - def_lines = ((line[0].strip(), line[1].strip()) for line in def_lines) - LAVA_CONFS = dict(def_lines) - #PANDA_DIR = LAVA_CONFS["PANDA_SRC_PATH"] - #PANDA_DIR = expandvars(PANDA_DIR) - LLVM_DIR = LAVA_CONFS["LLVM_SRC_PATH"] - LLVM_DIR = expandvars(LLVM_DIR) -except Exception: - LLVM_DIR = join(BUILD_DIR, "llvm-" + LLVM_VERSION) - -# Panda must the submodule -PANDA_DIR = abspath(join(LAVA_DIR, "panda/src")) -PANDA_BUILD_DIR = join(PANDA_DIR, '../build') - -# panda/scripts/install_ubuntu.sh -PANDA_UBUNTU = "https://raw.githubusercontent.com/panda-re/panda" \ - "/master/panda/scripts/install_ubuntu.sh" - -# libc6 needed for compiling btrace -# libjsoncpp needed for fbi json parsing -LAVA_DEPS = ["libjsoncpp-dev", "postgresql", "jq", "python-psycopg2", - "python-sqlalchemy", "socat", "libpq-dev", "cmake", - "docker.io", "bc", "python-pexpect", "python-psutil", - "python-lockfile", "genisoimage", "inotify-tools", - "build-essential", "python-pip", "libprotobuf-c0-dev", - "libodb-pgsql-2.4", "libfdt-dev"] - -PANDA_MAK = """ -# This is an autogenerated file from lava/setup.py. -PANDA_SRC_PATH := {PANDA_DIR} -PANDA_BUILD_DIR := {PANDA_DIR}/../build -""" -LLVM_MAK = """ -# This is an autogenerated file from lava/setup.py. -LLVM_SRC_PATH := {LLVM_SRC_PATH} -LLVM_BUILD_PATH := {LLVM_BUILD_PATH} -LLVM_BIN_PATH := $(LLVM_BUILD_PATH)/install/bin -""" - - -def progress(msg): - print('') -# PANDA_UBUNTU = "https://goo.gl/GNMNmJ" - print(Fore.GREEN + '[setup.py] ' + Fore.RESET + Style.BRIGHT - + msg + Style.RESET_ALL) - - -def error(msg): - print('') - print(Fore.RED + '[setup.py] ' + Fore.RESET + Style.BRIGHT - + msg + Style.RESET_ALL) - sys.exit(1) - - -def cmd_to_list(cmd): - cmd_args = shlex.split(cmd) if isinstance(cmd, str) else cmd - cmd = subprocess.list2cmdline(cmd_args) - return cmd, cmd_args - - -def run(cmd): - cmd, cmd_args = cmd_to_list(cmd) - try: - progress("Running [{}] . . . ".format(cmd)) - subprocess.check_call(cmd_args) - except subprocess.CalledProcessError: - error("[{}] cmd did not execute properly.".format(cmd)) - raise - - -def user_in_docker(username): - # grep exits with 0 if pattern found, 1 otherwise - return not command_exited_nonzero("groups {} | grep docker" - .format(username)) - - -DOCKER_MAP_DIRS = [LAVA_DIR, os.environ['HOME']] -DOCKER_MAP_FILES = ['/etc/passwd', '/etc/group', - '/etc/shadow', '/etc/gshadow'] -map_dirs_dedup = [] -# quadratic but who cares -for d in DOCKER_MAP_DIRS: - add = True - for d2 in DOCKER_MAP_DIRS: - if d is not d2 and d.startswith(d2): - add = False - break - if add: - map_dirs_dedup.append(d) - -map_dirs_args = sum([['-v', '{0}:{0}'.format(d)] - for d in map_dirs_dedup], []) -map_files_args = sum([['-v', '{0}:{0}:ro'.format(d)] - for d in DOCKER_MAP_FILES], []) - -ENV_VARS = ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', - 'https_proxy', 'no_proxy'] -env_map = {k: os.environ[k] for k in ENV_VARS if k in os.environ} -env_var_args = sum([['-e', '{}={}'.format(k, v)] - for k, v in env_map.iteritems()], []) -build_args = sum([['--build-arg', '{}={}'.format(k, v)] - for k, v in env_map.iteritems()], []) - -ALREADY_IN_DOCKER_GROUP = user_in_docker(getpass.getuser()) - - -def run_docker(cmd, workdir=None): - cmd, cmd_args = cmd_to_list(cmd) - sudo_args = [] if ALREADY_IN_DOCKER_GROUP else ['sudo'] - if workdir: - cmd = "cd {} && ".format(workdir) + cmd - # Have to be sudo in case we just installed docker - # and don't have the group yet. - cmd_args = sudo_args + ['docker', 'run', '--rm'] + map_dirs_args + \ - map_files_args + env_var_args + \ - [DOCKER_NAME, 'su', '-l', getpass.getuser(), '-c', cmd] - try: - progress("Running in docker [{}] . . . ".format(cmd)) - print "[{}]".format(" ".join(cmd_args)) - subprocess.check_call(cmd_args) - except subprocess.CalledProcessError: - error("[{}] cmd did not execute properly.") - raise - - -def main(): - parser = argparse.ArgumentParser(description='Setup LAVA') - parser.add_argument('-s', '--skip_docker_build', action='store_true', - default=False, - help='Whether or not to skip building docker image') - parser.add_argument('-f', '--fast', action='store_true', default=False, - help='Whether or not to skip building \ - binutils and glibc') - args = parser.parse_args() - IGNORE_DOCKER = args.skip_docker_build - - progress("In LAVA dir at {}".format(LAVA_DIR)) - # check to make sure we are not running as root/sudo - if os.getuid() == 0: - error("sudo/root privileges detected. \ - Run as user!\nUSAGE: {}".format(sys.argv[0])) - - progress("Installing LAVA apt-get dependencies") - #if not all(map(is_package_installed, LAVA_DEPS)): - # run(['sudo', 'apt-get', '-y', 'install'] + LAVA_DEPS) - - # set up postgres authentication. - if not isfile(join(os.environ['HOME'], '.pgpass')): - postgres_depends = subprocess.check_output(['dpkg-query', '-W', '-f', - '${depends}', - 'postgresql']).splitlines() - postgres_pkg = [d for d in postgres_depends - if re.match(r'postgresql-[0-9]+.?[0-9]+', d)][0] - postgres_version = postgres_pkg.replace('postgresql-', '') - pg_hba = "/etc/postgresql/{}/main/pg_hba.conf".format(postgres_version) - postgres_password = 'postgrespostgres' - run(['sudo', 'sed', '-i.bak', '-E', - r's/^(local\s+all\s+postgres\s+)md5$/\1peer/', pg_hba]) - run("sudo service postgresql reload") - password_sql = "ALTER USER postgres WITH PASSWORD '{}';" \ - .format(postgres_password) - run(['sudo', '-u', 'postgres', 'psql', '-c', password_sql]) - pgpass = join(os.environ['HOME'], '.pgpass') - with open(pgpass, 'w') as f: - f.write('*:*:*:postgres:{}'.format(postgres_password)) - os.chmod(pgpass, stat.S_IRUSR | stat.S_IWUSR) - run(['sudo', 'sed', '-i.bak', '-E', - r's/^(local\s+all\s+postgres\s+)peer$/\1md5/', pg_hba]) - run("sudo service postgresql reload") - - # check that user has docker install and docker privileges - progress("Checking if user is in docker group") - if not ALREADY_IN_DOCKER_GROUP: - run(['sudo', 'usermod', '-a', '-G', 'docker', getpass.getuser()]) - - # check that user has the LAVA build docker vm build - # if not run python scripts/build-docker.py - if not IGNORE_DOCKER: - progress("Checking that {} docker is properly built" - .format(DOCKER_NAME)) - sudo_args = [] if ALREADY_IN_DOCKER_GROUP else ['sudo'] - run(sudo_args + ['docker', 'build', '-t', - DOCKER_NAME, join(LAVA_DIR, 'docker')] + build_args) - compile_cmd = ['cd', join(LAVA_DIR, 'tools', 'btrace'), - '&&', 'bash', 'compile.sh'] - run_docker(['bash', '-c', subprocess.list2cmdline(compile_cmd)]) - - progress("Initializing PANDA submodule") - run(['git', 'submodule', 'init']) - run(['git', 'submodule', 'update']) - - # check for location of panda in PANDA_DIR - # make sure that it is PANDA2 - progress("Checking for PANDA in " + PANDA_DIR) - if not isdir(PANDA_DIR) or \ - not isfile(join(LAVA_DIR, "tools", "fbi", "panda.mak")) or \ - not isfile(join(PANDA_BUILD_DIR, 'config.log')): - progress("Building PANDA in " + PANDA_BUILD_DIR) - try: - os.makedirs(PANDA_BUILD_DIR) - except OSError: - print "Warning: Panda build directory is already there" - os.chdir(PANDA_DIR) - # The dtc submodule no longer works through git://, so we replace it with https://. - run(['sed', '-i', 's|url = git://git.qemu-project.org/dtc.git|url = https://git.qemu-project.org/dtc.git|g', '.gitmodules']) - # sync the submodule (apply the url protocol change) - run(['git', 'submodule', 'sync']) - run(['git', 'submodule', 'update', '--init', 'dtc']) - os.chdir(PANDA_BUILD_DIR) - run_docker([join(PANDA_DIR, '../setup.sh')]) - os.chdir(LAVA_DIR) - # Compile lavaTool inside the docker container. - progress("Creating $LAVA_DIR/tools/lavaTool/config.mak") - with open("tools/lavaTool/config.mak", "w") as f: - LLVM_DOCKER_DIR = '/llvm-{}'.format(LLVM_VERSION) - f.write(LLVM_MAK.format(LLVM_BUILD_PATH=LLVM_DOCKER_DIR, - LLVM_SRC_PATH=LLVM_DOCKER_DIR)) - os.chdir(LAVA_DIR) - - if not isfile(join(LAVA_DIR, "fbi", "panda.mak")): - progress("Creating $LAVA_DIR/tools/fbi/panda.mak") - with open(join(LAVA_DIR, "tools/fbi/panda.mak"), "w") as f: - f.write(PANDA_MAK.format(PANDA_DIR=PANDA_DIR)) - - if not isfile(join(LAVA_DIR, "lava.mak")): - progress("Creating $LAVA_DIR/lava.mak") - with open("lava.mak", 'w') as f: - f.write(PANDA_MAK.format(PANDA_DIR=PANDA_DIR)) - f.write(LLVM_MAK.format(LLVM_BUILD_PATH=LLVM_DOCKER_DIR, - LLVM_SRC_PATH=LLVM_DOCKER_DIR)) - - # ----------------End .mak file stuff --------------------- - progress("Making each component of lava, fbi and lavaTool") - - os.chdir(LAVA_DIR) - run_docker(['python', 'setup_container.py'], workdir=LAVA_DIR) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/setup_container.py b/setup_container.py deleted file mode 100644 index 217c899f..00000000 --- a/setup_container.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -import os -import sys -import shlex -import subprocess -from colorama import Fore -from colorama import Style - - -LAVA_DIR = os.path.dirname(os.path.abspath(sys.argv[0])) -os.chdir(LAVA_DIR) - - -def progress(msg): - print('') -# PANDA_UBUNTU = "https://goo.gl/GNMNmJ" - print(Fore.GREEN + '[setup.py] ' + Fore.RESET + Style.BRIGHT - + msg + Style.RESET_ALL) - -def error(msg): - print('') - print(Fore.RED + '[setup.py] ' + Fore.RESET + Style.BRIGHT - + msg + Style.RESET_ALL) - sys.exit(1) - - -def cmd_to_list(cmd): - cmd_args = shlex.split(cmd) if isinstance(cmd, str) else cmd - cmd = subprocess.list2cmdline(cmd_args) - return cmd, cmd_args - -def run(cmd): - cmd, cmd_args = cmd_to_list(cmd) - try: - progress("Running [{}] . . . ".format(cmd)) - subprocess.check_call(cmd_args) - except subprocess.CalledProcessError: - error("[{}] cmd did not execute properly.".format(cmd)) - raise - -if __name__ == '__main__': - # Compile btrace - compile_cmd = ['cd', os.path.join(LAVA_DIR, 'tools', 'btrace'), - '&&', 'bash', 'compile.sh'] - run(['bash', '-c', subprocess.list2cmdline(compile_cmd)]) - # Compile lavaTool inside the docker container. - progress("Creating $LAVA_DIR/tools/lavaTool/config.mak") - - run(['rm', '-rf', os.path.join(LAVA_DIR, 'tools/build')]) - run(['mkdir', '-p', os.path.join(LAVA_DIR, 'tools/build')]) - run(['mkdir', '-p', os.path.join(LAVA_DIR, 'tools/install')]) - - run(['cmake', '-B{}'.format(os.path.join(LAVA_DIR, 'tools/build')), - '-H{}'.format(os.path.join(LAVA_DIR, 'tools')), - '-DCMAKE_INSTALL_PREFIX={}'.format(os.path.join(LAVA_DIR, - 'tools/install'))]) - run(['make','--no-print-directory','-j4', 'install', '-C', - os.path.join(LAVA_DIR, 'tools/build/lavaTool')]) - - # -----------Beginning .mak file stuff ------------------- - # I think this would be useful, but i'm seperating it out - # in case anyone thinks it's a bad idea - # the idea is that if someone wants llvm and panda installed in certain - # locations, they can make their lava.mak ahead of time - # then setup.py will parse it and configure the environmet to those specs - os.chdir(LAVA_DIR) - - # ----------------End .mak file stuff --------------------- - progress("Making each component of lava, fbi and lavaTool") - progress("Compiling fbi") - - os.chdir(os.path.join(LAVA_DIR, "tools/build")) - run("make --no-print-directory -j4 -C fbi install") - os.chdir(LAVA_DIR) diff --git a/setup_container.sh b/setup_container.sh index bf9acee6..509c62e8 100644 --- a/setup_container.sh +++ b/setup_container.sh @@ -33,4 +33,4 @@ make --no-print-directory -j4 install -C "${LAVA_DIR}/tools/build/lavaTool" progress "Compiling fbi" -make --no-print-directory -j4 install -C "${LAVA_DIR}/tools/build/fbi" \ No newline at end of file +make --no-print-directory -j4 install -C "${LAVA_DIR}/tools/build/fbi" diff --git a/setup_postgres.py b/setup_postgres.py deleted file mode 100644 index 8050251f..00000000 --- a/setup_postgres.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import re -import shlex -import stat -import subprocess - -# Setup postgres. Mostly copied from setup.py - - -def cmd_to_list(cmd): - cmd_args = shlex.split(cmd) if isinstance(cmd, str) else cmd - cmd = subprocess.list2cmdline(cmd_args) - return cmd, cmd_args - - -def run(cmd): - cmd, cmd_args = cmd_to_list(cmd) - try: - print("Running [{}] . . . ".format(cmd)) - subprocess.check_call(cmd_args) - except subprocess.CalledProcessError: - print("[{}] cmd did not execute properly.".format(cmd)) - raise - - -def main(): - if not os.path.isfile(os.path.join(os.environ['HOME'], '.pgpass')): - postgres_depends = subprocess.check_output(['dpkg-query', '-W', '-f', - '${depends}', - 'postgresql']).splitlines() - postgres_pkg = [d for d in postgres_depends - if re.match(r'postgresql-[0-9]+.?[0-9]+', d)][0] - postgres_version = postgres_pkg.replace('postgresql-', '') - pg_hba = "/etc/postgresql/{}/main/pg_hba.conf".format(postgres_version) - postgres_password = 'postgrespostgres' - run(['sudo', 'sed', '-i.bak', '-E', - r's/^(local\s+all\s+postgres\s+)md5$/\1peer/', pg_hba]) - run("sudo service postgresql reload") - password_sql = "ALTER USER postgres WITH PASSWORD '{}';" \ - .format(postgres_password) - run(['sudo', '-u', 'postgres', 'psql', '-c', password_sql]) - pgpass = os.path.join(os.environ['HOME'], '.pgpass') - with open(pgpass, 'w') as f: - f.write('*:*:*:postgres:{}'.format(postgres_password)) - os.chmod(pgpass, stat.S_IRUSR | stat.S_IWUSR) - run(['sudo', 'sed', '-i.bak', '-E', - r's/^(local\s+all\s+postgres\s+)peer$/\1md5/', pg_hba]) - run("sudo service postgresql reload") - - -if __name__ == "__main__": - main()