Skip to content

Commit d8b29c9

Browse files
nepalnepal
authored
Add PG upgrade script. Add missing catalog/sql files from 14.2 (#6307)
1 parent c9e20ce commit d8b29c9

20 files changed

+5857
-64
lines changed

‎ydb/library/yql/parser/pg_wrapper/copy_src.py

100644100755
+127-44
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
#!/usr/bin/env python
1+
#!/usr/bin/env python3
22
# -*- coding: utf-8 -*-
33

44
import os
5+
import sys
56
from shutil import Error, copy2, rmtree
67
import subprocess
78
from collections import defaultdict
@@ -34,6 +35,79 @@
3435
"enumRelOpts",
3536
"stringRelOpts"])
3637

38+
source_dirs = [
39+
"postgresql/src/backend",
40+
"postgresql/src/common",
41+
"postgresql/src/include",
42+
"postgresql/src/port",
43+
"postgresql/src/timezone",
44+
]
45+
46+
def is_inside_source_dirs(filename):
47+
for dir in source_dirs:
48+
if filename.startswith(dir):
49+
return True
50+
return False
51+
52+
no_copy_sources = [
53+
# not used (and linux/darwin-only)
54+
"postgresql/src/backend/port/posix_sema.c",
55+
"postgresql/src/backend/port/sysv_shmem.c",
56+
# not used, was excluded earlier
57+
"postgresql/src/include/utils/help_config.h",
58+
"postgresql/src/backend/utils/misc/help_config.c",
59+
# provided in libc_compat
60+
"postgresql/src/port/strlcat.c",
61+
"postgresql/src/port/strlcpy.c",
62+
63+
# unneded headers
64+
"postgresql/src/common/md5_int.h",
65+
"postgresql/src/common/sha1_int.h",
66+
"postgresql/src/common/sha2_int.h",
67+
"postgresql/src/include/common/connect.h",
68+
"postgresql/src/include/common/logging.h",
69+
"postgresql/src/include/common/restricted_token.h",
70+
"postgresql/src/include/fe_utils/",
71+
"postgresql/src/include/getopt_long.h",
72+
"postgresql/src/include/jit/llvmjit.h",
73+
"postgresql/src/include/jit/llvmjit_emit.h",
74+
"postgresql/src/include/libpq/be-gssapi-common.h",
75+
"postgresql/src/include/port/aix.h",
76+
"postgresql/src/include/port/atomics/arch-hppa.h",
77+
"postgresql/src/include/port/atomics/arch-ia64.h",
78+
"postgresql/src/include/port/atomics/arch-ppc.h",
79+
"postgresql/src/include/port/atomics/generic-acc.h",
80+
"postgresql/src/include/port/atomics/generic-sunpro.h",
81+
"postgresql/src/include/port/cygwin.h",
82+
"postgresql/src/include/port/darwin.h",
83+
"postgresql/src/include/port/freebsd.h",
84+
"postgresql/src/include/port/hpux.h",
85+
"postgresql/src/include/port/linux.h",
86+
"postgresql/src/include/port/netbsd.h",
87+
"postgresql/src/include/port/openbsd.h",
88+
"postgresql/src/include/port/pg_pthread.h",
89+
"postgresql/src/include/port/solaris.h",
90+
"postgresql/src/include/port/win32/dlfcn.h",
91+
"postgresql/src/include/port/win32.h",
92+
"postgresql/src/include/replication/pgoutput.h",
93+
"postgresql/src/include/snowball/",
94+
"postgresql/src/port/pthread-win32.h",
95+
]
96+
97+
def need_copy(filename):
98+
if not is_inside_source_dirs(filename):
99+
return False
100+
for prefix in no_copy_sources:
101+
if filename.startswith(prefix):
102+
return False
103+
return True
104+
105+
exclude_from_source_list = set([
106+
# platform-specific, explicitly added in ya.make
107+
"postgresql/src/port/pg_crc32c_sse42.c",
108+
"postgresql/src/port/pg_crc32c_sse42_choose.c",
109+
])
110+
37111
def fix_line(line, all_lines, pos):
38112
global inside_func
39113
global define_for_yylval
@@ -218,45 +292,52 @@ def mycopy2(src, dst):
218292
if line is not None:
219293
fdst.write(line)
220294

221-
def copytree(src, dst):
222-
names = os.listdir(src)
223-
os.makedirs(dst)
295+
def copy_and_patch_sources(src_dir):
224296
errors = []
225-
for name in names:
226-
if name == "help_config.c": continue
227-
srcname = os.path.join(src, name)
228-
dstname = os.path.join(dst, name)
229-
#print(srcname)
230-
try:
231-
if os.path.isdir(srcname):
232-
copytree(srcname, dstname)
297+
with open(os.path.join(src_dir, "src_files"), "r") as fd:
298+
for line in fd:
299+
name = line.strip()
300+
if not need_copy(name):
301+
continue
302+
srcname = os.path.join(src_dir, name)
303+
if name == "postgresql/src/include/pg_config.h":
304+
dstname = "postgresql/src/include/pg_config-linux.h"
233305
else:
234-
mycopy2(srcname, dstname)
235-
except OSError as why:
236-
errors.append((srcname, dstname, str(why)))
237-
except Error as err:
238-
errors.extend(err.args[0])
239-
if errors:
240-
raise Error(errors)
241-
242-
def make_sources_list():
243-
with open("../../../../../contrib/libs/postgresql/ya.make","r") as fsrc:
244-
with open("pg_sources.inc","w") as fdst:
245-
fdst.write("SRCS(\n")
246-
for line in fsrc:
247-
if line.startswith(" src/"):
248-
#print(line.strip())
249-
if "/help_config.c" in line: continue
250-
fdst.write(" postgresql/" + line.strip() + "\n")
251-
fdst.write(")\n")
252-
253-
def get_vars():
254-
s=subprocess.check_output("objdump ../../../../../contrib/libs/postgresql/libpostgres.a -tw",shell=True).decode("utf-8")
306+
dstname = name
307+
try:
308+
os.makedirs(os.path.dirname(dstname), mode=0o755, exist_ok=True)
309+
if os.path.islink(srcname):
310+
target_full = os.path.realpath(srcname)
311+
target = os.path.relpath(target_full, start=os.path.realpath(os.path.dirname(srcname)))
312+
with open(dstname, "w") as f:
313+
print('#include "' + target + '" /* inclink generated by yamaker */', file=f)
314+
else:
315+
mycopy2(srcname, dstname)
316+
except OSError as why:
317+
errors.append((srcname, dstname, str(why)))
318+
except Error as err:
319+
errors.extend(err.args[0])
320+
if errors:
321+
raise Error(errors)
322+
323+
def make_sources_list(build_dir):
324+
with open(f"{build_dir}/src_files","r") as fsrc:
325+
with open("pg_sources.inc","w") as fdst:
326+
fdst.write("SRCS(\n")
327+
for line in fsrc:
328+
#print(line.strip())
329+
name = line.strip()
330+
if name.endswith(".c") and need_copy(name) and name not in exclude_from_source_list:
331+
fdst.write(" " + name + "\n")
332+
fdst.write(")\n")
333+
334+
def get_vars(build_dir):
335+
s=subprocess.check_output(f"objdump {build_dir}/postgresql/src/backend/postgres.a -tw",shell=True).decode("utf-8")
255336
for a in s.replace("\t"," ").split("\n"):
256337
for b in a.split(" "):
257338
sym=None
258339
if b.startswith(".bss."): sym=b[5:]
259-
elif b.startswith(".data."): sym=b[6:]
340+
elif b.startswith(".data.") and not b.startswith(".data.rel.ro."): sym=b[6:]
260341
if sym is not None:
261342
all_vars.add(sym.replace("yql_",""))
262343

@@ -300,23 +381,25 @@ def write_thread_inits():
300381
301382
void pg_thread_init(void) {
302383
if (pg_thread_init_flag) return;
303-
pg_thread_init_flag=1;
304-
setup_pg_thread_cleanup();
305-
pg_timezone_initialize();
306-
""", file=f)
384+
pg_thread_init_flag=1;""", file=f)
307385

308386
for a in sorted(thread_funcs):
309387
print(" " + a + "();", file=f)
310-
print("}", file=f)
388+
print("""
389+
setup_pg_thread_cleanup();
390+
pg_timezone_initialize();
391+
}""", file=f)
311392

312393
with open("thread_inits.h","w") as f:
313394
print("#pragma once", file=f)
314395
print("extern void pg_thread_init();", file=f)
315396

316397
if __name__ == "__main__":
317-
get_vars()
318-
make_sources_list()
319-
if os.path.isdir("postgresql"):
320-
rmtree("postgresql")
321-
copytree("../../../../../contrib/libs/postgresql", "postgresql")
398+
if len(sys.argv) != 2:
399+
print("Usage ", sys.argv[0], " <build directory>");
400+
sys.exit(1)
401+
build_dir=sys.argv[1]
402+
get_vars(build_dir)
403+
make_sources_list(build_dir)
404+
copy_and_patch_sources(build_dir)
322405
write_thread_inits()

‎ydb/library/yql/parser/pg_wrapper/copy_src.sh

+55-9
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,58 @@
11
#!/usr/bin/env bash
2-
set -eux
3-
echo copy data files...
4-
cp ../../../../../contrib/libs/postgresql/src/include/catalog/*.dat postgresql/src/include/catalog/
5-
cp ../../../../../contrib/libs/postgresql/src/backend/catalog/*.sql postgresql/src/backend/catalog/
2+
set -eu
3+
4+
VERSION="14.2"
5+
6+
errexit() {
7+
echo $1
8+
exit 1
9+
}
10+
11+
12+
13+
DIST="postgresql-$VERSION"
614

7-
rm -f ../../../../../contrib/libs/postgresql/libpostgres.a
8-
ya make ../../../../../contrib/libs/postgresql --checkout
15+
mkdir -p build
16+
cd build
17+
18+
echo cleanup
19+
rm -rf postgresql $DIST src_files
20+
rm -rf $DIST.tar.gz
21+
echo downloading sources
22+
wget -O $DIST.tar.gz "https://ftp.postgresql.org/pub/source/v$VERSION/$DIST.tar.gz" || errexit "Download failed"
23+
tar zxf $DIST.tar.gz
24+
mv $DIST postgresql
25+
cd postgresql
26+
echo patching postgresql sources
27+
patch -p0 < ../../source.patch || errexit "Source patching failed"
28+
29+
COMPILER=$(ya tool cc --print-path)
30+
TOOL_DIR=$(dirname $COMPILER)
31+
export PATH="$TOOL_DIR:$PATH"
32+
export CC=clang
33+
export AR=llvm-ar
34+
export CFLAGS="-ffunction-sections -fdata-sections"
35+
36+
echo configuring
37+
./configure \
38+
--with-openssl --with-icu --with-libxml --enable-debug --without-systemd \
39+
--without-gssapi --with-lz4 --with-ossp-uuid \
40+
|| errexit "Configure failed"
41+
echo building
42+
make -C src/backend all || errexit "Compilation failed"
43+
cd ..
44+
echo collecting *.c file list
45+
for i in $(find postgresql -name "*.o" | sed -e 's/o$/c/'); do if [ -f $i ]; then realpath --relative-to=. $i; fi; done > src_files
46+
echo collecting *.h file list
47+
find postgresql -type f -name "*.h" | sort >> src_files
48+
sort src_files > src_files.s
49+
mv src_files.s src_files
50+
51+
echo copy data files...
52+
rm -f ../postgresql/src/include/catalog/*.dat ../postgresql/src/backend/catalog/*.sql
53+
cp postgresql/src/include/catalog/*.dat ../postgresql/src/include/catalog/
54+
cp postgresql/src/backend/catalog/*.sql ../postgresql/src/backend/catalog/
955

10-
export LANG=ru_RU.UTF-8
11-
python3 copy_src.py
12-
for i in $(ls patches/); do (cd postgresql && patch -p1 < ../patches/$i); done
56+
cd ..
57+
echo copy and patch src files...
58+
python3 copy_src.py build

‎ydb/library/yql/parser/pg_wrapper/generate_patch.sh

-11
This file was deleted.

0 commit comments

Comments
 (0)