python-emscripten  Check-in [58e7feba3b]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Python3 initial support
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:58e7feba3be6f1ccb7c71aefdc7a75b78f1a24689fe47934a2b2951693dbec8b
User & Date: Beuc 2020-05-08 16:44:30
Context
2020-05-08
16:47
webprompt: drop py2 remark check-in: 08c4e36826 user: Beuc tags: trunk
16:44
Python3 initial support check-in: 58e7feba3b user: Beuc tags: trunk
16:26
Simplify Python compilation a bit check-in: 6319bb7bc7 user: Beuc tags: trunk
Changes

Changes to .fossil-settings/ignore-glob.

1
2
3
4
5
6

7
8
*/build/
*/crosspython-static/
*/crosspython-dynamic/
*/destdir
*/package/
*/Python-*.tgz

*/setuptools-*.zip
*/t/






>


1
2
3
4
5
6
7
8
9
*/build/
*/crosspython-static/
*/crosspython-dynamic/
*/destdir
*/package/
*/Python-*.tgz
*/Python-*.tar.xz
*/setuptools-*.zip
*/t/

Added 3.8/package-pythonhome.sh.











































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#!/bin/bash -e

# Creates a minimal Python file hierarchy at $PACKAGEDIR

# Copyright (C) 2018, 2019, 2020 Sylvain Beucler

# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.  This file is offered as-is,
# without any warranty.

FILE_PACKAGER="python $(dirname $(which emcc))/tools/file_packager.py"

PREFIX=${PREFIX:-$(dirname $(readlink -f $0))/destdir}
PACKAGEDIR=${PACKAGEDIR:-$(dirname $(readlink -f $0))/package}
OUTDIR=${OUTDIR:-.}
CROSSPYTHON=$(dirname $(readlink -f $0))/crosspython-static/bin/python3

# Python home

# optional lz4 compression, requires '-s LZ4=1'
LZ4=
if [ "$1" == "--lz4" ]; then LZ4="--lz4"; shift; fi

rm -rf $PACKAGEDIR/
mkdir -p $PACKAGEDIR

# Hard-coded modules: for 'print("hello, world.")'
# $@: additional, app-specific modules
for i in site.py os.py stat.py posixpath.py genericpath.py abc.py encodings/__init__.py codecs.py encodings/aliases.py encodings/utf_8.py io.py _collections_abc.py _sitebuiltins.py encodings/ascii.py encodings/latin_1.py \
    "$@"; do
    # TODO: no .pyo in Py3
    if [ $PREFIX/lib/python3.8/$i -nt $PREFIX/lib/python3.8/${i%.py}.pyo ]; then
	(cd $PREFIX && $CROSSPYTHON -OO -m py_compile lib/python3.8/$i)
    fi
    mkdir -p $PACKAGEDIR/lib/python3.8/$(dirname $i)
    #cp -au $PREFIX/lib/python3.8/${i%.py}.pyo $PACKAGEDIR/lib/python3.8/${i%.py}.py
    cp -au $PREFIX/lib/python3.8/${i%.py}.py $PACKAGEDIR/lib/python3.8/${i%.py}.py
done
# Large and leaks build paths, clean it:
#echo 'build_time_vars = {}' > $PACKAGEDIR/lib/python3.8/_sysconfigdata.py
#(cd $PACKAGEDIR && $CROSSPYTHON -OO -m py_compile lib/python3.8/_sysconfigdata.py)
#rm -f $PACKAGEDIR/lib/python3.8/_sysconfigdata.py

# --no-heap-copy: suited for ALLOW_MEMORY_GROWTH=1
PACKAGEDIR_FULLPATH=$(readlink -f $PACKAGEDIR)
(
    cd $OUTDIR;  # use relative path in xxx-data.js
    $FILE_PACKAGER \
	pythonhome.data --js-output=pythonhome-data.js \
	--preload $PACKAGEDIR_FULLPATH@/ \
	--use-preload-cache --no-heap-copy $LZ4
)

Added 3.8/patches/disable-set_inheritable.patch.









































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
Description: Disable set_inheritable
Forwarded: no
Author: Sylvain Beucler <beuc@beuc.net>
Last-Update: 2020-05-08

'set_inheritable' requires iotclt FIOCLEX/21585/0x5451 which is not supported:
RuntimeError: abort(bad ioctl syscall 21585) at jsStackTrace@http://localhost:6931/index.js:1955:17

--- a/Python/fileutils.c	2020-05-08 16:44:14.563517953 +0200
+++ b/Python/fileutils.c	2020-05-08 16:44:48.875389584 +0200
@@ -1091,6 +1091,9 @@
 static int
 set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
 {
+#ifdef EMSCRIPTEN
+return 0;
+#endif
 #ifdef MS_WINDOWS
     HANDLE handle;
     DWORD flags;

Added 3.8/patches/python3-cross_compile.patch.



































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
Description: Fix build system for Emscripten cross-compilation.
Forwarded: no
Author: Sylvain Beucler <beuc@beuc.net>
Last-Update: 2020-05-08

--- a/config.sub	2020-05-08 12:15:20.097297249 +0200
+++ b/config.sub	2020-05-08 12:17:19.704298501 +0200
@@ -1394,7 +1394,7 @@
 	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
 	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
 	      | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox* | -bme* \
-	      | -midnightbsd*)
+	      | -midnightbsd* | -emscripten*)
 	# Remember, each alternative MUST END IN *, to match a version number.
 		;;
 	-qnx*)
--- a/configure	2020-02-24 22:36:25.000000000 +0100
+++ b/configure	2020-05-08 12:41:34.048250804 +0200
@@ -3274,6 +3274,9 @@
 	*-*-vxworks*)
 	    ac_sys_system=VxWorks
 	    ;;
+	*-*-emscripten)
+		ac_sys_system=Emscripten
+		;;
 	*)
 		# for now, limit cross builds to known configurations
 		MACHDEP="unknown"
@@ -3324,6 +3327,9 @@
 	*-*-vxworks*)
 		_host_cpu=$host_cpu
 		;;
+	*-*-emscripten)
+		ac_sys_system=Emscripten
+		;;
 	*)
 		# for now, limit cross builds to known configurations
 		MACHDEP="unknown"
diff --git a/config-emscripten.site b/config-emscripten.site
new file mode 100644
index 0000000..c273024
--- /dev/null
+++ b/config-emscripten.site
@@ -0,0 +1,5 @@
+# Required by ./configure
+ac_cv_file__dev_ptmx=no
+ac_cv_file__dev_ptc=no
+# Support dynamic linking
+ac_cv_func_dlopen=yes

Added 3.8/patches/series.





>
>
1
2
python3-cross_compile.patch
disable-set_inheritable.patch

Added 3.8/python.sh.































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
#!/bin/bash -ex

# Compile minimal Python for Emscripten and native local testing

# Copyright (C) 2018, 2019, 2020  Sylvain Beucler

# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.  This file is offered as-is,
# without any warranty.

VERSION=3.8.2
SCRIPTDIR=$(dirname $(readlink -f $0))
DESTDIR=${DESTDIR:-$SCRIPTDIR/destdir}
SETUPLOCAL=${SETUPLOCAL:-'/dev/null'}

CACHEROOT=$SCRIPTDIR
BUILD=$SCRIPTDIR/build
export QUILT_PATCHES=$(dirname $(readlink -f $0))/patches

WGET=${WGET:-wget}

unpack () {
    $WGET -c https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tar.xz -P $CACHEROOT/
    mkdir -p $BUILD
    cd $BUILD/
    rm -rf Python-$VERSION/
    tar xf $CACHEROOT/Python-$VERSION.tar.xz
    cd Python-$VERSION/
    quilt push -a
}

# use cases:
# - python/pgen/.pyo for emscripten() below
# - common basis for crosspython below
hostpython () {
    cd $BUILD/Python-$VERSION/
    mkdir -p native
    (
        cd native/
        if [ ! -e config.status ]; then
            ../configure \
                --prefix=''
        fi

        make -j$(nproc)
        make install DESTDIR=$BUILD/hostpython
    )
}

emscripten () {
    cd $BUILD/Python-$VERSION/
    mkdir -p emscripten
    (
        cd emscripten/
        # OPT=-Oz: TODO
        # CONFIG_SITE: deals with cross-compilation https://bugs.python.org/msg136962
        # --without-pymalloc: ?
        # --disable-ipv6: ?
	# --disable-shared: compile statically for Emscripten perfs + incomplete PIC support
        if [ ! -e config.status ]; then
            CONFIG_SITE=../config-emscripten.site BASECFLAGS='-s USE_ZLIB=1' READELF=true \
                PYTHON_FOR_BUILD=$BUILD/Python-$VERSION/native/python \
                emconfigure ../configure \
                --host=asmjs-unknown-emscripten --build=$(../config.guess) \
                --prefix='' \
                --without-pymalloc --disable-ipv6 \
                --disable-shared
        fi
	cat <<EOF >> pyconfig.h
/* issues with posixmodule.c */
#undef HAVE_POSIX_SPAWN
#undef HAVE_POSIX_SPAWNP
/* issues with signalmodule.c */
#undef HAVE_PTHREAD_SIGMASK
EOF

        # Modules/Setup.local
        echo '*static*' > Modules/Setup.local
        cat $SETUPLOCAL >> Modules/Setup.local
        # drop -I/-L/-lz, we USE_ZLIB=1 (keep it in SETUPLOCAL for mock)
        sed -i -e 's/^\(zlib zlibmodule.c\).*/\1/' Modules/Setup.local
	# drop system include_dirs for cross-compilation
        sed -i -e 's/ self.add_multiarch_paths()/ #&/' ../setup.py
        # decrease .pyo size by dropping docstrings
        sed -i -e '/compileall.py/ s/ -O / -OO /' Makefile
    
        emmake make -j$(nproc)
        emmake make install DESTDIR=$DESTDIR

        # Basic trimming
        # Disabled for now, better cherry-pick the files we need
        #emmake make install DESTDIR=$(pwd)/destdir
        #find destdir/ -name "*.py" -print0 | xargs -r0 rm
        #find destdir/ -name "*.pyo" -print0 | xargs -r0 rm  # only keep .pyc, .pyo apparently don't work
        #find destdir/ -name "*.so" -print0 | xargs -r0 rm
        #rm -rf destdir/usr/local/bin/
        #rm -rf destdir/usr/local/share/man/
        #rm -rf destdir/usr/local/include/
        #rm -rf destdir/usr/local/lib/*.a
        #rm -rf destdir/usr/local/lib/pkgconfig/
        #rm -rf destdir/usr/local/lib/python2.7/test/
        # Ditch .so for now, they cause an abort() with dynamic
        # linking unless we recompile all of them as SIDE_MODULE-s
        #rm -rf $DESTDIR/lib/python2.7/lib-dynload/
    )
}

# For mock-ing emscripten environment through static desktop python
mock () {
    cd $BUILD/Python-$VERSION/
    mkdir -p native
    (
        cd native/
        if [ ! -e config.status ]; then
            ../configure \
                --prefix=$BUILD/hostpython/ \
                --without-threads --without-pymalloc --without-signal-module --disable-ipv6 \
                --disable-shared
        fi
        echo '*static*' > Modules/Setup.local
        cat $SETUPLOCAL >> Modules/Setup.local

        make -j$(nproc) Parser/pgen python
    
        make -j$(nproc)
        DESTDIR= make install
    )
}

# python aimed at compiling third-party Python modules to WASM
# - building static/dynamic wasm modules
# - compiling .pyo files, in emscripten() and package-xxx.sh
# Uses hostpython; detects its PYTHONHOME through ../lib AFAICS, no need to recompile
# Usage:
# .../crosspython-static/bin/python  setup.py xxx --root=.../destdir/ --prefix=''
# .../crosspython-dynamic/bin/python setup.py xxx --root=.../destdir/ --prefix=''
# .../crosspython-static/bin/python -OO -m py_compile xxx.py
# Note: maybe create and patch two virtualenv instead?  but harder to
#   use with dual static/dynamic Makefile; doesn't handle root=destdir
crosspython () {
    cd $SCRIPTDIR
    # Copy-link hostpython except for include/ and
    # lib/python3.8/_sysconfigdata.py
    for variant in static dynamic; do
	rm -rf crosspython-$variant
	mkdir crosspython-$variant
	(
	    cd crosspython-$variant
	    for i in $(cd ../build/hostpython && ls -A); do
		ln -s ../build/hostpython/$i $i
	    done
	    rm include lib
	    mkdir lib
	    for i in $(cd ../build/hostpython/lib && ls -A); do
		ln -s ../../build/hostpython/lib/$i lib/$i
	    done
	    rm lib/python3.8
	    mkdir lib/python3.8
	    for i in $(cd ../build/hostpython/lib/python3.8 && ls -A); do
		ln -s ../../../build/hostpython/lib/python3.8/$i lib/python3.8/$i
	    done
	    
	    # Use Python.h configured for WASM
	    ln -s $DESTDIR/include include
	    
	    # TODO: Use compiler settings configured for WASM
	    #rm lib/python3.8/_sysconfigdata.*
	    #cp -a $DESTDIR/lib/python3.8/_sysconfigdata.* lib/python3.8/
	)
    done
    # TODO: 'CCSHARED': 'xxx',
    #sed -i -e "s/'CCSHARED': .*/'CCSHARED': '-fPIC -s SIDE_MODULE=1',/" \
    #   crosspython-dynamic/lib/python3.8/_sysconfigdata.py
}

case "$1" in
    unpack|hostpython|emscripten|mock|crosspython)
        "$1"
        ;;
    '')
        unpack
        hostpython
        emscripten
        crosspython
        ;;
    *)
        echo "Usage: $0 unpack|native|emscripten"
        exit 1
        ;;
esac

Added 3.8/webprompt.sh.













































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#!/bin/bash -ex

# Simple Python prompt for the browser, for smoke testing

# Copyright (C) 2019, 2020  Sylvain Beucler

# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.  This file is offered as-is,
# without any warranty.

# Alternatively: use Emscripten's old binary:
# emscripten/tests/python/python.bc -s ERROR_ON_UNDEFINED_SYMBOLS=0

INSTALLDIR=${INSTALLDIR:-$(dirname $(readlink -f $0))/destdir}
BUILD=t

mkdir -p $BUILD

cython -3 ../emscripten.pyx -o $BUILD/emscripten.c
cython -3 ../emscripten_fetch.pyx -o $BUILD/emscripten_fetch.c
# utf_32_be: support Unicode characters e.g. u'é'
# __future__.py: for emscripten.pyx
PREFIX=$INSTALLDIR OUTDIR=$BUILD ./package-pythonhome.sh \
    encodings/utf_32_be.py __future__.py

FLAGS='-O3'
while (( $# )); do
    case "$1" in
	debug) FLAGS='-s ASSERTIONS=1 -g -s FETCH_DEBUG=1';;
	async) ASYNC='-s ASYNCIFY=1 -O3';;
    esac
    shift
done
emcc -o $BUILD/index.html \
  ../webprompt-main.c $BUILD/emscripten.c $BUILD/emscripten_fetch.c \
  $FLAGS \
  -I$INSTALLDIR/include/python3.8 -L$INSTALLDIR/lib -lpython3.8 \
  -s EMULATE_FUNCTION_POINTER_CASTS=1 \
  -s USE_ZLIB=1 \
  -s FETCH=1 \
  -s ALLOW_MEMORY_GROWTH=1 \
  -s FORCE_FILESYSTEM=1 -s RETAIN_COMPILER_SETTINGS=1 \
  $ASYNC \
  --shell-file ../webprompt-shell.html -s MINIFY_HTML=0 \
  -s EXPORTED_FUNCTIONS='[_main, _Py_Initialize, _PyRun_SimpleString, _pyruni]' \
  -s EXTRA_EXPORTED_RUNTIME_METHODS='[ccall, cwrap]'

# emrun --serve_after_close t/index.html

# cython -3 ../mock/emscripten.pyx -o t/mock.c
# cython -3 ../mock/emscripten_fetch.pyx -o t/mock2.c
# gcc -g -I build/hostpython/include/python3.8 -L build/hostpython/lib/ t/mock.c t/mock2.c ../webprompt-main.c -lpython3.8 -ldl -lm -lutil -lz -lpthread
# PYTHONHOME=build/hostpython/ ./a.out

Changes to webprompt-main.c.

9
10
11
12
13
14
15







16
17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

38
39
40
41
42
43
44
 * offered as-is, without any warranty.
 */

#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif
#include <Python.h>







PyMODINIT_FUNC initemscripten(void);
PyMODINIT_FUNC initemscripten_fetch(void);


// Run a line *and* display the result
// PyRun_StringFlags only returns a non-None object with Py_eval_input (no 'print' support)
// PyRun_InteractiveOne always reads stdin even with another 'fp', so we redirect stdin
void pyruni() {
  freopen("/tmp/input.py", "rb", stdin);
  PyRun_InteractiveOne(stdin, "<stdin>");
}

int main(int argc, char**argv) {
  Py_OptimizeFlag = 2; // look for .pyo rather than .pyc
  Py_FrozenFlag   = 1; // drop <exec_prefix> warnings
  Py_VerboseFlag  = 1; // trace modules loading
  Py_InitializeEx(0);  // 0 = get rid of 'Calling stub instead of sigaction()'
  static struct _inittab builtins[] = {
    { "emscripten", initemscripten },
    { "emscripten_fetch", initemscripten_fetch },
    {NULL, NULL}
  };
  PyImport_ExtendInittab(builtins);

#ifdef __EMSCRIPTEN__
  emscripten_exit_with_live_runtime();
#else
  pyruni();
#endif
  return 0;
}







>
>
>
>
>
>
>
|
<
>













<

|
|



>







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

24
25
26
27
28
29
30
31
32
33
34
35
36
37

38
39
40
41
42
43
44
45
46
47
48
49
50
51
 * offered as-is, without any warranty.
 */

#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif
#include <Python.h>

#if PY_MAJOR_VERSION >= 3
#define MODINIT(name) PyInit_##name
#else
#define MODINIT(name) init##name
#endif

PyMODINIT_FUNC MODINIT(emscripten) (void);

PyMODINIT_FUNC MODINIT(emscripten_fetch) (void);

// Run a line *and* display the result
// PyRun_StringFlags only returns a non-None object with Py_eval_input (no 'print' support)
// PyRun_InteractiveOne always reads stdin even with another 'fp', so we redirect stdin
void pyruni() {
  freopen("/tmp/input.py", "rb", stdin);
  PyRun_InteractiveOne(stdin, "<stdin>");
}

int main(int argc, char**argv) {
  Py_OptimizeFlag = 2; // look for .pyo rather than .pyc
  Py_FrozenFlag   = 1; // drop <exec_prefix> warnings
  Py_VerboseFlag  = 1; // trace modules loading

  static struct _inittab builtins[] = {
    { "emscripten", MODINIT(emscripten) },
    { "emscripten_fetch", MODINIT(emscripten_fetch) },
    {NULL, NULL}
  };
  PyImport_ExtendInittab(builtins);
  Py_InitializeEx(0);  // 0 = get rid of 'Calling stub instead of sigaction()'
#ifdef __EMSCRIPTEN__
  emscripten_exit_with_live_runtime();
#else
  pyruni();
#endif
  return 0;
}