python-emscripten  Check-in [814dcfc8ca]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix crosspython
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:814dcfc8ca3cf187e3790c12374df790fe60d02167254ede3393767132154d7c
User & Date: Beuc 2020-06-06 14:17:39
Context
2020-06-06
16:15
Update copyright check-in: 1a1988d51c user: Beuc tags: trunk
14:17
Fix crosspython check-in: 814dcfc8ca user: Beuc tags: trunk
2020-05-27
17:58
3.8: size/perf-optimized python code check-in: acc37a2322 user: Beuc tags: trunk
Changes

Changes to 2.7.10/python.sh.

62
63
64
65
66
67
68


69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
..
92
93
94
95
96
97
98
99
100


101
102
103
104
105

106
107
108
109
110
111
112
...
197
198
199
200
201
202
203
204
205
206
207
    cd $BUILD/Python-$VERSION/
    mkdir -p emscripten
    (
        cd emscripten/
        # OPT=-Oz: TODO
        # CONFIG_SITE: deals with cross-compilation https://bugs.python.org/msg136962
        # not needed as emcc has a single arch: BASECFLAGS=-m32 LDFLAGS=-m32


        # --without-threads: pthreads experimental as of 2020-05
        #   cf. https://emscripten.org/docs/porting/pthreads.html
        # --without-signal-module: no process signals in Emscripten
        # --without-pymalloc: ?
        # --disable-ipv6: browser-side networking
        # --disable-shared: compile statically for Emscripten perfs + incomplete PIC support
        if [ ! -e config.status ]; then
            CONFIG_SITE=../config.site BASECFLAGS='-s USE_ZLIB=1' \
                PYTHON_FOR_BUILD=$BUILD/Python-$VERSION/native/python \
                emconfigure ../configure \
                --host=asmjs-unknown-emscripten --build=$(../config.guess) \
                --prefix='' \
                --without-threads --without-pymalloc --without-signal-module --disable-ipv6 \
                --disable-shared
        fi
	# Fix bad detection
        sed -i -e 's,^#define HAVE_GCC_ASM_FOR_X87.*,/* & */,' pyconfig.h

        # pgen native setup
        # note: need to build 'pgen' once before overwriting it with the native one
        # note: PGEN=../native/Parser/pgen doesn't work, make overwrites it
        emmake make Parser/pgen
        \cp --preserve=mode ../native/Parser/pgen Parser/
................................................................................

        # Modules/Setup.local
        echo '*static*' > Modules/Setup.local
        cat $SETUPLOCAL >> Modules/Setup.local
        # drop -I/-L/-lz, we USE_ZLIB=1 (keep it in SETUPLOCAL for mock)
        sed -i -e 's/^\(zlib zlibmodule.c\).*/\1/' Modules/Setup.local
    
        emmake make -j$(nproc)



        # setup.py install_lib doesn't respect DESTDIR
        echo -e 'sharedinstall:\n\ttrue' >> Makefile
        # decrease .pyo size by dropping docstrings
        sed -i -e '/compileall.py/ s/ -O / -OO /' Makefile
        emmake make install DESTDIR=$DESTDIR


        # Basic trimming
        # Disabled for now, better cherry-pick the files we need
        #emmake make install DESTDIR=$(pwd)/destdir
        #find destdir/ -name "*.py" -print0 | xargs -r0 rm
        #find destdir/ -name "*.pyo" -print0 | xargs -r0 rm  # only keep .pyc, .pyo apparently don't work
        #find destdir/ -name "*.so" -print0 | xargs -r0 rm
................................................................................
    '')
        unpack
        hostpython
        emscripten
        crosspython
        ;;
    *)
        echo "Usage: $0 unpack|native|emscripten"
        exit 1
        ;;
esac







>
>








|






|







 







<
|
>
>
|
|
|
|
|
>







 







|



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
..
94
95
96
97
98
99
100

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
...
201
202
203
204
205
206
207
208
209
210
211
    cd $BUILD/Python-$VERSION/
    mkdir -p emscripten
    (
        cd emscripten/
        # OPT=-Oz: TODO
        # CONFIG_SITE: deals with cross-compilation https://bugs.python.org/msg136962
        # not needed as emcc has a single arch: BASECFLAGS=-m32 LDFLAGS=-m32
        # PATH: detect our Python, beware of conflict with emcc's python3
        #       don't use PYTHON_FOR_BUILD which is high-level / lots of options
        # --without-threads: pthreads experimental as of 2020-05
        #   cf. https://emscripten.org/docs/porting/pthreads.html
        # --without-signal-module: no process signals in Emscripten
        # --without-pymalloc: ?
        # --disable-ipv6: browser-side networking
        # --disable-shared: compile statically for Emscripten perfs + incomplete PIC support
        if [ ! -e config.status ]; then
            CONFIG_SITE=../config.site BASECFLAGS='-s USE_ZLIB=1' \
                PATH=$BUILD/Python-$VERSION/native:$PATH \
                emconfigure ../configure \
                --host=asmjs-unknown-emscripten --build=$(../config.guess) \
                --prefix='' \
                --without-threads --without-pymalloc --without-signal-module --disable-ipv6 \
                --disable-shared
        fi
        # Fix bad detection
        sed -i -e 's,^#define HAVE_GCC_ASM_FOR_X87.*,/* & */,' pyconfig.h

        # pgen native setup
        # note: need to build 'pgen' once before overwriting it with the native one
        # note: PGEN=../native/Parser/pgen doesn't work, make overwrites it
        emmake make Parser/pgen
        \cp --preserve=mode ../native/Parser/pgen Parser/
................................................................................

        # Modules/Setup.local
        echo '*static*' > Modules/Setup.local
        cat $SETUPLOCAL >> Modules/Setup.local
        # drop -I/-L/-lz, we USE_ZLIB=1 (keep it in SETUPLOCAL for mock)
        sed -i -e 's/^\(zlib zlibmodule.c\).*/\1/' Modules/Setup.local
    

        (
            export PATH=$BUILD/Python-$VERSION/native:$PATH
            emmake make -j$(nproc)
            # setup.py install_lib doesn't respect DESTDIR
            echo -e 'sharedinstall:\n\ttrue' >> Makefile
            # decrease .pyo size by dropping docstrings
            sed -i -e '/compileall.py/ s/ -O / -OO /' Makefile
            emmake make install DESTDIR=$DESTDIR
        )

        # Basic trimming
        # Disabled for now, better cherry-pick the files we need
        #emmake make install DESTDIR=$(pwd)/destdir
        #find destdir/ -name "*.py" -print0 | xargs -r0 rm
        #find destdir/ -name "*.pyo" -print0 | xargs -r0 rm  # only keep .pyc, .pyo apparently don't work
        #find destdir/ -name "*.so" -print0 | xargs -r0 rm
................................................................................
    '')
        unpack
        hostpython
        emscripten
        crosspython
        ;;
    *)
        echo "Usage: $0 unpack|hostpython|emscripten|mock|crosspython"
        exit 1
        ;;
esac

Changes to 3.8/python.sh.

52
53
54
55
56
57
58


59
60
61
62
63
64
65
66
67
68
69
70
71
..
80
81
82
83
84
85
86


87
88
89
90
91

92
93
94
95
96
97
98
...
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
emscripten () {
    cd $BUILD/Python-$VERSION/
    mkdir -p emscripten
    (
        cd emscripten/
        # OPT=-Oz: TODO
        # CONFIG_SITE: deals with cross-compilation https://bugs.python.org/msg136962


        # --without-pymalloc: ?
        # --disable-ipv6: ?
	# --disable-shared: compile statically for Emscripten perfs + incomplete PIC support
        if [ ! -e config.status ]; then
            CONFIG_SITE=../config-emscripten.site BASECFLAGS='-s USE_ZLIB=1' READELF=true \
                PYTHON_FOR_BUILD=$BUILD/Python-$VERSION/native/python \
                emconfigure ../configure \
                --host=asmjs-unknown-emscripten --build=$(../config.guess) \
                --prefix='' \
                --without-pymalloc --disable-ipv6 \
                --disable-shared
        fi
	cat <<EOF >> pyconfig.h
................................................................................
        echo '*static*' > Modules/Setup.local
        cat $SETUPLOCAL >> Modules/Setup.local
        # drop -I/-L/-lz, we USE_ZLIB=1 (keep it in SETUPLOCAL for mock)
        sed -i -e 's/^\(zlib zlibmodule.c\).*/\1/' Modules/Setup.local
        # decrease .pyo size by dropping docstrings
        sed -i -e '/compileall.py/ s/ -O / -OO /' Makefile



        # Trigger setup.py:CROSS_COMPILING (introduced in 3.8)
        export _PYTHON_HOST_PLATFORM=emscripten

        emmake make -j$(nproc)
        emmake make install DESTDIR=$DESTDIR


        # Basic trimming
        # Disabled for now, better cherry-pick the files we need
        #emmake make install DESTDIR=$(pwd)/destdir
        #find destdir/ -name "*.py" -print0 | xargs -r0 rm
        #find destdir/ -name "*.pyo" -print0 | xargs -r0 rm  # only keep .pyc, .pyo apparently don't work
        #find destdir/ -name "*.so" -print0 | xargs -r0 rm
................................................................................
	    for i in $(cd ../build/hostpython/lib/python3.8 && ls -A); do
		ln -s ../../../build/hostpython/lib/python3.8/$i lib/python3.8/$i
	    done
	    
	    # Use Python.h configured for WASM
	    ln -s $DESTDIR/include include
	    
	    # TODO: Use compiler settings configured for WASM
	    #rm lib/python3.8/_sysconfigdata.*
	    #cp -a $DESTDIR/lib/python3.8/_sysconfigdata.* lib/python3.8/
	)
    done
    # TODO: 'CCSHARED': 'xxx',
    #sed -i -e "s/'CCSHARED': .*/'CCSHARED': '-fPIC -s SIDE_MODULE=1',/" \
    #   crosspython-dynamic/lib/python3.8/_sysconfigdata.py
}

case "$1" in
    unpack|hostpython|emscripten|mock|crosspython)
        "$1"
        ;;
    '')
        unpack
        hostpython
        emscripten
        crosspython
        ;;
    *)
        echo "Usage: $0 unpack|native|emscripten"
        exit 1
        ;;
esac







>
>


|


|







 







>
>
|
|
<
|
|
>







 







|
|
|


|
|
|













|



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
..
82
83
84
85
86
87
88
89
90
91
92

93
94
95
96
97
98
99
100
101
102
...
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
emscripten () {
    cd $BUILD/Python-$VERSION/
    mkdir -p emscripten
    (
        cd emscripten/
        # OPT=-Oz: TODO
        # CONFIG_SITE: deals with cross-compilation https://bugs.python.org/msg136962
        # PATH: detect our Python, beware of conflict with emcc's python3
        #       don't use PYTHON_FOR_BUILD which is high-level / lots of options
        # --without-pymalloc: ?
        # --disable-ipv6: ?
        # --disable-shared: compile statically for Emscripten perfs + incomplete PIC support
        if [ ! -e config.status ]; then
            CONFIG_SITE=../config-emscripten.site BASECFLAGS='-s USE_ZLIB=1' READELF=true \
                PATH=$BUILD/Python-$VERSION/native:$PATH \
                emconfigure ../configure \
                --host=asmjs-unknown-emscripten --build=$(../config.guess) \
                --prefix='' \
                --without-pymalloc --disable-ipv6 \
                --disable-shared
        fi
	cat <<EOF >> pyconfig.h
................................................................................
        echo '*static*' > Modules/Setup.local
        cat $SETUPLOCAL >> Modules/Setup.local
        # drop -I/-L/-lz, we USE_ZLIB=1 (keep it in SETUPLOCAL for mock)
        sed -i -e 's/^\(zlib zlibmodule.c\).*/\1/' Modules/Setup.local
        # decrease .pyo size by dropping docstrings
        sed -i -e '/compileall.py/ s/ -O / -OO /' Makefile

        (
            export PATH=$BUILD/Python-$VERSION/native:$PATH
            # Trigger setup.py:CROSS_COMPILING (introduced in 3.8)
            export _PYTHON_HOST_PLATFORM=emscripten

            emmake make -j$(nproc)
            emmake make install DESTDIR=$DESTDIR
        )

        # Basic trimming
        # Disabled for now, better cherry-pick the files we need
        #emmake make install DESTDIR=$(pwd)/destdir
        #find destdir/ -name "*.py" -print0 | xargs -r0 rm
        #find destdir/ -name "*.pyo" -print0 | xargs -r0 rm  # only keep .pyc, .pyo apparently don't work
        #find destdir/ -name "*.so" -print0 | xargs -r0 rm
................................................................................
	    for i in $(cd ../build/hostpython/lib/python3.8 && ls -A); do
		ln -s ../../../build/hostpython/lib/python3.8/$i lib/python3.8/$i
	    done
	    
	    # Use Python.h configured for WASM
	    ln -s $DESTDIR/include include
	    
	    # Use compiler settings configured for WASM
	    cp -a $DESTDIR/lib/python3.8/_sysconfigdata__emscripten_.py \
	       lib/python3.8/_sysconfigdata_*.py
	)
    done
    # 'CCSHARED': 'xxx',
    sed -i -e "s/'CCSHARED': .*/'CCSHARED': '-fPIC -s SIDE_MODULE=1',/" \
       crosspython-dynamic/lib/python3.8/_sysconfigdata_*.py
}

case "$1" in
    unpack|hostpython|emscripten|mock|crosspython)
        "$1"
        ;;
    '')
        unpack
        hostpython
        emscripten
        crosspython
        ;;
    *)
        echo "Usage: $0 unpack|hostpython|emscripten|mock|crosspython"
        exit 1
        ;;
esac