1: #!/bin/bash
     2: #
     3: # zbuild: this is the phased building script used to develop *ZyX
     4: #
     5: # more docs/functionality coming soon(ish)...
     6: # 
     7: # todo: argument processing for (and usage) (see below/eof for comments)
     8: #
     9: 
    10: # f11 performance issue, theoretically solved by a 'feature' in f12
    11: #qcow_too_slow=1
    12: # also solved by qemu-0.11.1 
    13: qcow_too_slow=0
    14: 
    15: 
    16: #
    17: # defaults to be overridden
    18: #
    19: 
    20: # overrides md5verify and prelink (to off)
    21: quickbuild=0
    22: # mirrorcheck is not yet implemented
    23: domirrorcheck=0
    24: dosynthesis=0
    25: doupdate=0
    26: domutate=0
    27: doxmutate=0
    28: dobake=0
    29: # useful for debugging smirfgen development
    30: dosmirf=0
    31: doxmog=0
    32: # gathers srpms and does checksums
    33: dopackage=0
    34: 
    35: #
    36: # current overrides (need to be script args)
    37: #
    38: 
    39: quickbuild=1
    40: #domirrorcheck=1
    41: #dosynthesis=1
    42: #doupdate=1
    43: domutate=1
    44: dobake=1
    45: #dosmirf=1
    46: doxmog=1
    47: #dopackage=1
    48: 
    49: #strain_name=F-ZyX
    50: #strain_name=Fork-ZyX
    51: #strain_name=G-ZyX
    52: strain_name=Guitar-ZyX
    53: #strain_name=D-ZyX
    54: #strain_name=min
    55: #strain_version="0.9.3"
    56: strain_version="0.5.0"
    57: #mirrorlist=/var/mirrors/f12/viros.mirrors.cfg
    58: #mirrorlist=/mnt/big/pub/org/info/mirrors/f12/viros.mirrors.cfg
    59: mirrorlist=/mnt/zbuild/mirrors/f12/viros.mirrors.cfg
    60: 
    61: 
    62: workarea=$( pwd )
    63: 
    64: strain="${strain_name}-${strain_version}"
    65: 
    66: datestamp=$( date +%Y-%m-%d--%H-%M-%S )
    67: 
    68: rundir=$( pwd )
    69: progdir=$( pushd $( dirname $0 ) > /dev/null 2>&1 ; \
    70:     pwd ; popd > /dev/null 2>&1 )
    71: virosdir=$( pushd $( dirname $0 )/.. > /dev/null 2>&1 ; \
    72:     pwd ; popd > /dev/null 2>&1 )
    73: 
    74: 
    75: builddir="${workarea}/build.${datestamp}"
    76: mkdir -p $builddir
    77: cd $builddir
    78: 
    79: find ${virosdir} -name "*~" -exec rm -vf '{}' ';'
    80: find ${virosdir} -name "#*" -exec rm -vf '{}' ';'
    81: 
    82: if (($dosynthesis)); then
    83: #    --verbose --debug \
    84:     time \
    85: 	${virosdir}/viros synthesize \
    86: 	--verbose --debug \
    87: 	--mirrorlist=${mirrorlist} \
    88: 	--tmpdir=${builddir} \
    89: 	--config=${strain} \
    90: 	--vsysgen_screencast_output=${strain}.screencast \
    91: 	--output=${builddir}/${strain}.vsi \
    92: 	2>&1 | tee -a ${builddir}/zmakeit.phased.log
    93:     rm -f ${rundir}/${strain}.vsi
    94:     ln -fs ${builddir}/${strain}.vsi \
    95: 	${rundir}/${strain}.vsi
    96: else
    97:     ln -s $( readlink -f ${rundir}/${strain}.vsi ) \
    98: 	${builddir}/${strain}.vsi
    99: fi
   100: 
   101: if (($doupdate)); then
   102: 
   103:     qemu-img create -b ${strain}.vsi \
   104: 	-f qcow2 ${strain}-updated.vsi 
   105: 
   106:     if (($qcow_too_slow)); then
   107: 	rm -f ${strain}-updated.vsi 
   108: 	cp -av --sparse=always \
   109: 	    $( readlink -f ${strain}.vsi ) \
   110: 	    ${strain}-updated.vsi 
   111:     fi
   112: 
   113:     time \
   114: 	${virosdir}/viros mutate \
   115: 	--verbose --debug \
   116: 	--mirrorlist=${mirrorlist} \
   117: 	--tmpdir=${builddir} \
   118: 	--traits=splice-compose.repos.f12::no_mutate_prepost=1:mode=install \
   119: 	--traits=splice-kupdate \
   120: 	--traits=splice-software::update=yes \
   121: 	--traits=splice-compose.repos.f12::no_mutate_prepost=1:mode=uninstall \
   122: 	${builddir}/${strain}-updated.vsi \
   123: 	2>&1 | tee -a ${builddir}/zmakeit.phased.log
   124:     rm -f ${rundir}/${strain}-updated.vsi
   125:     ln -fs ${builddir}/${strain}-updated.vsi \
   126: 	${rundir}/${strain}-updated.vsi
   127: else
   128:     ln -s $( readlink -f ${rundir}/${strain}-updated.vsi ) \
   129: 	${builddir}/${strain}-updated.vsi
   130: fi
   131: 
   132: if (($domutate)); then
   133: 
   134:     qemu-img create -b ${strain}-updated.vsi \
   135: 	-f qcow2 ${strain}-mutated.vsi 
   136: 
   137:     if (($qcow_too_slow)); then
   138: 	rm -f ${strain}-mutated.vsi 
   139: 	cp -av --sparse=always \
   140: 	    $( readlink -f ${strain}-updated.vsi ) \
   141: 	    ${strain}-mutated.vsi 
   142:     fi
   143: 
   144: 
   145:     if (($quickbuild)); then
   146: 	xvmutopts="--traits=unsplice-restorecon"
   147:     else
   148: 	xvmutopts="--traits=splice-noop"
   149:     fi
   150: 
   151:     time \
   152: 	${virosdir}/viros mutate \
   153: 	--verbose --debug \
   154: 	--mirrorlist=${mirrorlist} \
   155: 	--tmpdir=${builddir} \
   156: 	--config=${strain} \
   157: 	"${xvmutopts}" \
   158: 	${builddir}/${strain}-mutated.vsi \
   159: 	2>&1 | tee -a ${builddir}/zmakeit.phased.log
   160:     rm -f ${rundir}/${strain}-mutated.vsi
   161:     ln -fs ${builddir}/${strain}-mutated.vsi \
   162: 	${rundir}/${strain}-mutated.vsi
   163: else
   164:     ln -s $( readlink -f ${rundir}/${strain}-mutated.vsi ) \
   165: 	${builddir}/${strain}-mutated.vsi
   166: fi
   167: 
   168: if (($dobake)); then
   169:     xdone=0
   170:     while ((!${xdone})); do
   171: 
   172: 	iter_datestamp=$( date +%Y-%m-%d--%H-%M-%S )
   173: 	mkdir ${builddir}/vtmp.iter.${iter_datestamp}
   174: 
   175: 	if [ -f ${strain}-bake.img ]; then
   176: 	    mv ${strain}-bake.img ${strain}-bake.${iter_datestamp}.img
   177: 	fi
   178: 
   179: 	if [ -f ${strain}-mutated-cooked.vsi ]; then
   180: 	    mv ${strain}-mutated-cooked.vsi ${strain}-mutated-cooked.${iter_datestamp}.vsi
   181: 	fi
   182: 
   183: 	qemu-img create ${strain}-bake.img 1234M
   184: 	qemu-img create \
   185: 	    -b ${strain}-mutated.vsi \
   186: 	    -f qcow2 \
   187: 	    ${strain}-mutated-cooked.vsi
   188: 
   189: 	if (($qcow_too_slow)); then
   190: 	    rm -f ${strain}-mutated-cooked.vsi 
   191: 	    cp -av --sparse=always \
   192: 		$( readlink -f ${strain}-mutated.vsi ) \
   193: 	    ${strain}-mutated-cooked.vsi 
   194: 	fi
   195: 
   196: 	# builddir is probably not getting expanded here, eval perhaps
   197: 	# now seems like it may be expanced, but not what we want in the
   198: 	# retry scenario, i.e. it kills the new one as well
   199: 	eval "( sleep 3600 ; kill -KILL \$( < ${builddir}/vtmp.iter.${iter_datestamp}/vtmp.qrap*/qpid.txt ) ) &"
   200: 
   201: 	
   202: 	time \
   203: 	    env \
   204: 	    vopt_debug=1 \
   205: 	    vopt_verbose=1 \
   206: 	    viros_stupid_vncname="VirOS___Simulation___Monitor" \
   207: 	    TMPDIR=${builddir}/vtmp.iter.${iter_datestamp} \
   208: 	    ${virosdir}/viros qrap \
   209: 	    --vdisplay=X \
   210: 	    --reaper=true \
   211: 	    --recvideo=./${strain}-bakecast \
   212: 	    -net none -m 512 -boot c \
   213: 	    -full-screen \
   214: 	    -hda ./${strain}-mutated-cooked.vsi \
   215: 	    -hdb ./${strain}-bake.img
   216: 	rm -f ${rundir}/${strain}-mutated-cooked.vsi
   217: 	ln -fs ${builddir}/${strain}-mutated-cooked.vsi \
   218: 	    ${rundir}/${strain}-mutated-cooked.vsi
   219: 
   220: 	if ( tar tvf ./${strain}-bake.img | grep -q "output/guest.cooked/.mozilla/firefox/.*\.default/prefs.js" ); then
   221: 	    if ( tar tvf ./${strain}-bake.img | grep -q "output/guest.cooked/.mozilla/firefox/.*\.default/extensions.ini" ); then
   222: 		xdone=1
   223: 	    fi
   224: 	fi
   225:     done
   226: 
   227: else
   228:     ln -s $( readlink -f ${rundir}/${strain}-mutated.vsi ) \
   229: 	${builddir}/${strain}-mutated-cooked.vsi
   230:     ln -s $( readlink -f ${rundir}/${strain}-bake.img ) \
   231: 	${builddir}/${strain}-bake.img
   232: fi
   233: 
   234: if (($dosmirf)); then
   235:     time \
   236: 	${virosdir}/viros transmogrify \
   237: 	--verbose --debug \
   238: 	--tmpdir=${builddir} \
   239: 	--config=${strain} \
   240: 	zyx-smirf \
   241: 	${builddir}/${strain}-mutated-cooked.vsi \
   242: 	${builddir}/${strain}.smirf \
   243: 	2>&1 | tee -a ${builddir}/zmakeit.phased.log
   244:     rm -f ${rundir}/${strain}.smirf
   245:     ln -fs ${builddir}/${strain}.smirf \
   246: 	${rundir}/${strain}.smirf
   247: fi
   248: 
   249: if (($doxmog)); then
   250: 
   251:     # things to override config values with
   252:     xvxmogopts='--vxmogopts=bogus=0'
   253:     if (($quickbuild)); then
   254: 	xvxmogopts+=" --vxmogopts=md5verify=0"
   255: 	xvxmogopts+=" --vxmogopts=prelink=0"
   256: 	xvxmogopts+=" --vxmogopts=timestamps=none"
   257:     fi
   258: 
   259:     time \
   260: 	${virosdir}/viros transmogrify \
   261: 	--verbose --debug \
   262: 	--tmpdir=${builddir} \
   263: 	--config=${strain} \
   264: 	${xvxmogopts} \
   265: 	zyx-live \
   266: 	${builddir}/${strain}-mutated-cooked.vsi \
   267: 	${builddir}/${strain}.iso \
   268: 	2>&1 | tee -a ${builddir}/zmakeit.phased.log
   269:     rm -f ${rundir}/${strain}.iso
   270:     ln -fs ${builddir}/${strain}.iso \
   271: 	${rundir}/${strain}.iso
   272: else
   273:     ln -s $( readlink -f ${rundir}/${strain}.iso ) \
   274: 	${builddir}/${strain}.iso
   275: fi
   276: 
   277: if (($dopackage)); then
   278: 
   279:     rm -rf ./goldbits
   280:     mkdir ./goldbits
   281:     pushd ./goldbits
   282: 
   283:     golden_iso=$( readlink -f "${rundir}/${strain}.iso" )
   284:     golden_iso_name=$( basename "${golden_iso}" )
   285:     golden_iso_dir=$( dirname "${golden_iso}" )
   286: 
   287:     mutated_vsi=$( readlink -f "${rundir}/${strain}-mutated.vsi" )
   288:     mutated_vsi_name=$( basename "${mutated_vsi}" )
   289:     mutated_vsi_dir=$( dirname "${mutated_vsi}" )
   290: 
   291:     mutated_cooked_vsi=$( readlink -f "${rundir}/${strain}-mutated-cooked.vsi" )
   292:     mutated_cooked_vsi_name=$( basename "${mutated_cooked_vsi}" )
   293:     mutated_cooked_vsi_dir=$( dirname "${mutated_cooked_vsi}" )
   294: 
   295:     mirror_dir=$( dirname "${mirrorlist}" )
   296: 
   297:     mkdir ./${strain}.srpms
   298: 
   299:     echo "extracting srpm list from baked image ..."
   300:     tar xf "${mutated_cooked_vsi_dir}/${strain}-bake.img" output/sysinfo.srpms
   301:     mv output/sysinfo.srpms "./${strain}.srpms.txt"
   302:     rmdir output
   303:     
   304:     echo "collecting srpms ..."
   305:     for srpm in $( < "./${strain}.srpms.txt" ); do
   306: 	find "${mirror_dir}" -name "${srpm}" -exec cp -a '{}' "./${strain}.srpms/" ';'
   307:     done
   308: 
   309:     echo "creating checksums for srpms ..."
   310:     sha512sum "./${strain}.srpms/"* > \
   311: 	"./${strain}.srpms.sha512sums.txt"
   312:     
   313:     # the .. is to facilitate the case when the phased run had to be split across multiple builds
   314:     echo "copying build logs to goldbits/${strain}.build.log ..."
   315:     cat "${builddir}"/../build.*/zmakeit.phased.log >> \
   316: 	"./${strain}.build.log"
   317: 
   318:     echo "copying output live iso to goldbits..."
   319:     cp -a "${golden_iso}" "./${strain}.iso"
   320: 
   321:     echo "running checksum of liveiso, and other bits to be embedded in livesrc iso..."
   322:     sha512sum * > "./${strain}.sha512sums.txt"
   323: 
   324:     echo "cloning live iso to livesrc iso..."
   325:     cp -a "${golden_iso}" "./${strain}.wsource.iso"
   326: 
   327:     echo "creating source appendage for livesrc iso..."
   328:     tar cf - "./${strain}.srpms" *sha512sums* \
   329: 	>> "./${strain}.wsource.append.tar"
   330: 
   331:     cat "./${strain}.sha512sums.txt" \
   332: 	>> "./${strain}.wsource.append.tar"
   333: 
   334:     echo "appending source in reverse to livesrc iso..."
   335:     tac "./${strain}.wsource.append.tar" >> "./${strain}.wsource.iso"
   336: 
   337:     rm -rf "./${strain}.srpms" *sha512sums* 
   338: 
   339:     echo "final checksumming of live and livesrc iso in progress..."
   340:     sha512sum \
   341: 	"./${strain}.iso" \
   342: 	"./${strain}.wsource.append.tar" \
   343: 	"./${strain}.wsource.iso" \
   344: 	"./${strain}.build.log" \
   345: 	>> "./${strain}.sha512sum.txt"
   346: 
   347:     bzip2 "./${strain}.build.log" 
   348: 
   349:     popd
   350: 
   351:     rm -f ${rundir}/${strain}.goldbits
   352:     ln -fs ${builddir}/goldbits \
   353: 	${rundir}/${strain}.goldbits
   354: 
   355: fi
   356: 
   357: # TODO
   358: #
   359: #            --sig=<sigdir>
   360: #            [--check-keys=<check_key_dir>]
   361: #            --check-mirror
   362: #            [--mirrorlist=/var/mirrors/f10/viros.mirrors.cfg]
   363: #
   364: #            --nox (hide viros synth monitor)
   365: #            --reuse-synthesis
   366: #            --reuse-update
   367: #            --reuse-mutate
   368: #            --reuse-bake
   369: #            --reuse-xmog
   370: #            --strain=
   371: #
   372: # - possibly need retry for synth and mutate
   373: # - disk space checking - check for 31G to be on the safe side
   374: #  start with a make release, remaking the viros trait, then copying elsewhere.
   375: # - save traits.tar in packaging, for nonsavetraits case
   376: # - (use builddep to expand) 
   377: #  - just pass over rpmqa doing query on each packages src,
   378: #    - then iter again skipping already done packages,
   379: #       - iter till no new packages
   380: #
   381: