shared: Introduce new top-level dir and move 1st party lib code there.

This commit moves all first-party code developed for this project from lib/
to shared/, so that lib/ now only contains third-party code.

The following directories are moved as-is from lib to shared:

    lib/libc            -> shared/libc
    lib/memzip          -> shared/memzip
    lib/netutils        -> shared/netutils
    lib/timeutils       -> shared/timeutils
    lib/upytesthelper   -> shared/upytesthelper

All files in lib/embed/ have been moved to shared/libc/.

lib/mp-readline has been moved to shared/readline.

lib/utils has been moved to shared/runtime, with the exception of
lib/utils/printf.c which has been moved to shared/libc/printf.c.

Signed-off-by: Damien George <damien@micropython.org>

6 files changed, 332 insertions, 0 deletions

diff --git a/shared/memzip/README.md b/shared/memzip/README.md
new file mode 100644
index 000000000..287d0fc48
--- /dev/null
+++ b/shared/memzip/README.md
@@ -0,0 +1,28 @@
+MEMZIP - a simple readonly file system
+
+memzip takes a zip file which is comprised of uncompressed files and
+and presents it as a filesystem, allowing Python files to be imported.
+
+The script make-memzip.py takes a directory name and will create a zip file
+containing uncompressed files found in the directory. It will then generate
+a C file which contains the data from the zip file.
+
+A typical addition to a makefile would look like:
+```
+SRC_C += \
+    lib/memzip/import.c \
+    lib/memzip/lexermemzip.c \
+    lib/memzip/memzip.c \
+
+OBJ += $(BUILD)/memzip-files.o
+
+MAKE_MEMZIP = ../lib/memzip/make-memzip.py
+
+$(BUILD)/memzip-files.o: $(BUILD)/memzip-files.c
+    $(call compile_c)
+
+$(BUILD)/memzip-files.c: $(shell find ${MEMZIP_DIR} -type f)
+    @$(ECHO) "Creating $@"
+    $(Q)$(PYTHON) $(MAKE_MEMZIP) --zip-file $(BUILD)/memzip-files.zip --c-file $@ $(MEMZIP_DIR)
+```
+
diff --git a/shared/memzip/import.c b/shared/memzip/import.c
new file mode 100644
index 000000000..2d5225b88
--- /dev/null
+++ b/shared/memzip/import.c
@@ -0,0 +1,17 @@
+#include <stdio.h>
+
+#include "py/lexer.h"
+#include "memzip.h"
+
+mp_import_stat_t mp_import_stat(const char *path) {
+    MEMZIP_FILE_INFO info;
+
+    if (memzip_stat(path, &info) != MZ_OK) {
+        return MP_IMPORT_STAT_NO_EXIST;
+    }
+
+    if (info.is_dir) {
+        return MP_IMPORT_STAT_DIR;
+    }
+    return MP_IMPORT_STAT_FILE;
+}
diff --git a/shared/memzip/lexermemzip.c b/shared/memzip/lexermemzip.c
new file mode 100644
index 000000000..6b26961bd
--- /dev/null
+++ b/shared/memzip/lexermemzip.c
@@ -0,0 +1,19 @@
+#include <stdlib.h>
+
+#include "py/lexer.h"
+#include "py/runtime.h"
+#include "py/mperrno.h"
+#include "memzip.h"
+
+mp_lexer_t *mp_lexer_new_from_file(const char *filename)
+{
+    void *data;
+    size_t len;
+
+    if (memzip_locate(filename, &data, &len) != MZ_OK) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+
+    return mp_lexer_new_from_str_len(qstr_from_str(filename), (const char *)data, (mp_uint_t)len, 0);
+}
+
diff --git a/shared/memzip/make-memzip.py b/shared/memzip/make-memzip.py
new file mode 100755
index 000000000..9730f5e00
--- /dev/null
+++ b/shared/memzip/make-memzip.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+#
+# Takes a directory of files and zips them up (as uncompressed files).
+# This then gets converted into a C data structure which can be read
+# like a filesystem at runtime.
+#
+# This is somewhat like frozen modules in python, but allows arbitrary files
+# to be used.
+
+from __future__ import print_function
+
+import argparse
+import os
+import subprocess
+import sys
+import types
+
+def create_zip(zip_filename, zip_dir):
+    abs_zip_filename = os.path.abspath(zip_filename)
+    save_cwd = os.getcwd()
+    os.chdir(zip_dir)
+    if os.path.exists(abs_zip_filename):
+        os.remove(abs_zip_filename)
+    subprocess.check_call(['zip', '-0', '-r', '-D', abs_zip_filename, '.'])
+    os.chdir(save_cwd)
+
+def create_c_from_file(c_filename, zip_filename):
+    with open(zip_filename, 'rb') as zip_file:
+        with open(c_filename, 'wb') as c_file:
+            print('#include <stdint.h>', file=c_file)
+            print('', file=c_file)
+            print('const uint8_t memzip_data[] = {', file=c_file)
+            while True:
+                buf = zip_file.read(16)
+                if not buf:
+                    break
+                print('   ', end='', file=c_file)
+                for byte in buf:
+                    if type(byte) is types.StringType:
+                        print(' 0x{:02x},'.format(ord(byte)), end='', file=c_file)
+                    else:
+                        print(' 0x{:02x},'.format(byte), end='', file=c_file)
+                print('', file=c_file)
+            print('};', file=c_file)
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='make-memzip.py',
+        usage='%(prog)s [options] [command]',
+        description='Generates a C source memzip file.'
+    )
+    parser.add_argument(
+        '-z', '--zip-file',
+        dest='zip_filename',
+        help='Specifies the name of the created zip file.',
+        default='memzip_files.zip'
+    )
+    parser.add_argument(
+        '-c', '--c-file',
+        dest='c_filename',
+        help='Specifies the name of the created C source file.',
+        default='memzip_files.c'
+    )
+    parser.add_argument(
+        dest='source_dir',
+        default='memzip_files'
+    )
+    args = parser.parse_args(sys.argv[1:])
+
+    print('args.zip_filename =', args.zip_filename)
+    print('args.c_filename =', args.c_filename)
+    print('args.source_dir =', args.source_dir)
+
+    create_zip(args.zip_filename, args.source_dir)
+    create_c_from_file(args.c_filename, args.zip_filename)
+
+if __name__ == "__main__":
+    main()
+
diff --git a/shared/memzip/memzip.c b/shared/memzip/memzip.c
new file mode 100644
index 000000000..3fbea8e1e
--- /dev/null
+++ b/shared/memzip/memzip.c
@@ -0,0 +1,106 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include "py/mpconfig.h"
+#include "py/misc.h"
+#include "memzip.h"
+
+extern uint8_t memzip_data[];
+
+const MEMZIP_FILE_HDR *memzip_find_file_header(const char *filename) {
+
+    const MEMZIP_FILE_HDR *file_hdr = (const MEMZIP_FILE_HDR *)memzip_data;
+    uint8_t *mem_data;
+
+    /* Zip file filenames don't have a leading /, so we strip it off */
+
+    if (*filename == '/') {
+        filename++;
+    }
+    while (file_hdr->signature == MEMZIP_FILE_HEADER_SIGNATURE) {
+        const char *file_hdr_filename = (const char *)&file_hdr[1];
+        mem_data = (uint8_t *)file_hdr_filename;
+        mem_data += file_hdr->filename_len;
+        mem_data += file_hdr->extra_len;
+        if (!strncmp(file_hdr_filename, filename, file_hdr->filename_len)) {
+            /* We found a match */
+            return file_hdr;
+        }
+        mem_data += file_hdr->uncompressed_size;
+        file_hdr = (const MEMZIP_FILE_HDR *)mem_data;
+    }
+    return NULL;
+}
+
+bool memzip_is_dir(const char *filename) {
+    const MEMZIP_FILE_HDR *file_hdr = (const MEMZIP_FILE_HDR *)memzip_data;
+    uint8_t *mem_data;
+
+    if (strcmp(filename, "/") == 0) {
+        // The root directory is a directory.
+        return true;
+    }
+
+    // Zip filenames don't have a leading /, so we strip it off
+    if (*filename == '/') {
+        filename++;
+    }
+    size_t filename_len = strlen(filename);
+
+    while (file_hdr->signature == MEMZIP_FILE_HEADER_SIGNATURE) {
+        const char *file_hdr_filename = (const char *)&file_hdr[1];
+        if (filename_len < file_hdr->filename_len &&
+            strncmp(file_hdr_filename, filename, filename_len) == 0 &&
+            file_hdr_filename[filename_len] == '/') {
+            return true;
+        }
+
+        mem_data = (uint8_t *)file_hdr_filename;
+        mem_data += file_hdr->filename_len;
+        mem_data += file_hdr->extra_len;
+        mem_data += file_hdr->uncompressed_size;
+        file_hdr = (const MEMZIP_FILE_HDR *)mem_data;
+    }
+    return NULL;
+
+}
+
+MEMZIP_RESULT memzip_locate(const char *filename, void **data, size_t *len)
+{
+    const MEMZIP_FILE_HDR *file_hdr = memzip_find_file_header(filename);
+    if (file_hdr == NULL) {
+        return MZ_NO_FILE;
+    }
+    if (file_hdr->compression_method != 0) {
+        return MZ_FILE_COMPRESSED;
+    }
+
+    uint8_t *mem_data;
+    mem_data = (uint8_t *)&file_hdr[1];
+    mem_data += file_hdr->filename_len;
+    mem_data += file_hdr->extra_len;
+
+    *data = mem_data;
+    *len = file_hdr->uncompressed_size;
+    return MZ_OK;
+}
+
+MEMZIP_RESULT memzip_stat(const char *path, MEMZIP_FILE_INFO *info) {
+    const MEMZIP_FILE_HDR *file_hdr = memzip_find_file_header(path);
+    if (file_hdr == NULL) {
+        if (memzip_is_dir(path)) {
+            info->file_size = 0;
+            info->last_mod_date = 0;
+            info->last_mod_time = 0;
+            info->is_dir = 1;
+            return MZ_OK;
+        }
+        return MZ_NO_FILE;
+    }
+    info->file_size = file_hdr->uncompressed_size;
+    info->last_mod_date = file_hdr->last_mod_date;
+    info->last_mod_time = file_hdr->last_mod_time;
+    info->is_dir = 0;
+
+    return MZ_OK;
+}
diff --git a/shared/memzip/memzip.h b/shared/memzip/memzip.h
new file mode 100644
index 000000000..667e2df7e
--- /dev/null
+++ b/shared/memzip/memzip.h
@@ -0,0 +1,83 @@
+#pragma pack(push, 1)
+
+#define MEMZIP_FILE_HEADER_SIGNATURE 0x04034b50
+typedef struct
+{
+    uint32_t    signature;
+    uint16_t    version;
+    uint16_t    flags;
+    uint16_t    compression_method;
+    uint16_t    last_mod_time;
+    uint16_t    last_mod_date;
+    uint32_t    crc32;
+    uint32_t    compressed_size;
+    uint32_t    uncompressed_size;
+    uint16_t    filename_len;
+    uint16_t    extra_len;
+
+    /* char     filename[filename_len]  */
+    /* uint8_t  extra[extra_len]        */
+
+} MEMZIP_FILE_HDR;
+
+#define MEMZIP_CENTRAL_DIRECTORY_SIGNATURE 0x02014b50
+typedef struct
+{
+    uint32_t    signature;
+    uint16_t    version_made_by;
+    uint16_t    version_read_with;
+    uint16_t    flags;
+    uint16_t    compression_method;
+    uint16_t    last_mod_time;
+    uint16_t    last_mod_date;
+    uint32_t    crc32;
+    uint32_t    compressed_size;
+    uint32_t    uncompressed_size;
+    uint16_t    filename_len;
+    uint16_t    extra_len;
+    uint16_t    disk_num;
+    uint16_t    internal_file_attributes;
+    uint32_t    external_file_attributes;
+    uint32_t    file_header_offset;
+
+    /* char     filename[filename_len]  */
+    /* uint8_t  extra[extra_len]        */
+
+} MEMZIP_CENTRAL_DIRECTORY_HDR;
+
+#define MEMZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE 0x06054b50
+typedef struct
+{
+    uint32_t    signature;
+    uint16_t    disk_num;
+    uint16_t    central_directory_disk;
+    uint16_t    num_central_directories_this_disk;
+    uint16_t    total_central_directories;
+    uint32_t    central_directory_size;
+    uint32_t    central_directory_offset;
+    uint16_t    comment_len;
+
+    /* char     comment[comment_len]    */
+
+} MEMZIP_END_OF_CENTRAL_DIRECTORY;
+
+#pragma pack(pop)
+
+typedef enum {
+    MZ_OK = 0,          /* (0) Succeeded */
+    MZ_NO_FILE,         /* (1) Could not find the file. */
+    MZ_FILE_COMPRESSED, /* (2) File is compressed (expecting uncompressed) */
+
+} MEMZIP_RESULT;
+
+typedef struct {
+    uint32_t    file_size;
+    uint16_t    last_mod_date;
+    uint16_t    last_mod_time;
+    uint8_t     is_dir;
+
+} MEMZIP_FILE_INFO;
+
+MEMZIP_RESULT memzip_locate(const char *filename, void **data, size_t *len);
+
+MEMZIP_RESULT memzip_stat(const char *path, MEMZIP_FILE_INFO *info);