system/splitjob: Added (Tool to split up data).
Signed-off-by: Willy Sudiarto Raharjo <willysr@slackbuilds.org>
This commit is contained in:
parent
959ec73084
commit
f41d672514
|
@ -0,0 +1,28 @@
|
|||
This program is used to split up data from stdin in blocks which are
|
||||
sent as input to parallel invocations of commands. The output from
|
||||
those are then concatenated in the right order and sent to stdout.
|
||||
|
||||
Splitting up and parallelizing jobs like this might be useful to speed
|
||||
up compression using multiple CPU cores or even multiple computers.
|
||||
|
||||
For this approach to be useful, the compressed format needs to allow
|
||||
multiple compressed files to be concatenated. This is the case for
|
||||
gzip, bzip2, lzip and xz.
|
||||
|
||||
Example 1, use multiple logical cores:
|
||||
splitjob -j 4 bzip2 < bigfile > bigfile.bz2
|
||||
|
||||
Example 2, use remote machines:
|
||||
splitjob "ssh host1 gzip" "ssh host2 gzip" < f > f.gz
|
||||
|
||||
The above example assumes that ssh is configured to allow logins
|
||||
without asking for password. See the manpage for ssh-keygen or do
|
||||
a google search for examples on how to accomplish this.
|
||||
|
||||
Example 3, Use bigger blocks to reduce overhead:
|
||||
splitjob -j 2 -b 10M gzip < file > file.gz
|
||||
|
||||
For "xz -9" a block size of 384 MB gives best compression.
|
||||
|
||||
Example 4, parallel decompression:
|
||||
splitjob -X -r 10 -j 10 -b 384M "xz -d -" < file.xz > file
|
|
@ -0,0 +1,19 @@
|
|||
# HOW TO EDIT THIS FILE:
|
||||
# The "handy ruler" below makes it easier to edit a package description.
|
||||
# Line up the first '|' above the ':' following the base package name, and
|
||||
# the '|' on the right side marks the last column you can put a character in.
|
||||
# You must make exactly 11 lines for the formatting to be correct. It's also
|
||||
# customary to leave one space after the ':' except on otherwise blank lines.
|
||||
|
||||
|-----handy-ruler------------------------------------------------------|
|
||||
splitjob: splitjob (tool to split up data)
|
||||
splitjob:
|
||||
splitjob: Splitjob is tool to split up data and send it to parallel invocations
|
||||
splitjob: of some program and concatenate the output of those invocations.
|
||||
splitjob: It is useful to speed up compression of big amounts of data by using
|
||||
splitjob: multiple CPU cores or even multiple machines in the network.
|
||||
splitjob:
|
||||
splitjob:
|
||||
splitjob:
|
||||
splitjob:
|
||||
splitjob:
|
|
@ -0,0 +1,98 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Slackware build script for splitjob
|
||||
|
||||
# Copyright 2022 Henrik Carlqvist, Sweden
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use of this script, with or without modification, is
|
||||
# permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of this script must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR IMPLIED
|
||||
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
cd $(dirname $0) ; CWD=$(pwd)
|
||||
|
||||
PRGNAM=splitjob
|
||||
VERSION=${VERSION:-3.2}
|
||||
BUILD=${BUILD:-1}
|
||||
TAG=${TAG:-_SBo}
|
||||
PKGTYPE=${PKGTYPE:-tgz}
|
||||
|
||||
if [ -z "$ARCH" ]; then
|
||||
case "$( uname -m )" in
|
||||
i?86) ARCH=i586 ;;
|
||||
arm*) ARCH=arm ;;
|
||||
*) ARCH=$( uname -m ) ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if [ ! -z "${PRINT_PACKAGE_NAME}" ]; then
|
||||
echo "$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TMP=${TMP:-/tmp/SBo}
|
||||
PKG=$TMP/package-$PRGNAM
|
||||
OUTPUT=${OUTPUT:-/tmp}
|
||||
|
||||
if [ "$ARCH" = "i586" ]; then
|
||||
SLKCFLAGS="-O2 -march=i586 -mtune=i686"
|
||||
LIBDIRSUFFIX=""
|
||||
elif [ "$ARCH" = "i686" ]; then
|
||||
SLKCFLAGS="-O2 -march=i686 -mtune=i686"
|
||||
LIBDIRSUFFIX=""
|
||||
elif [ "$ARCH" = "x86_64" ]; then
|
||||
SLKCFLAGS="-O2 -fPIC"
|
||||
LIBDIRSUFFIX="64"
|
||||
else
|
||||
SLKCFLAGS="-O2"
|
||||
LIBDIRSUFFIX=""
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
rm -rf $PKG
|
||||
mkdir -p $TMP $PKG $OUTPUT
|
||||
cd $TMP
|
||||
rm -rf $PRGNAM-$VERSION
|
||||
tar xvf $CWD/$PRGNAM-$VERSION.tar.xz
|
||||
cd $PRGNAM-$VERSION
|
||||
chown -R root:root .
|
||||
find -L . \
|
||||
\( -perm 777 -o -perm 775 -o -perm 750 -o -perm 711 -o -perm 555 \
|
||||
-o -perm 511 \) -exec chmod 755 {} \; -o \
|
||||
\( -perm 666 -o -perm 664 -o -perm 640 -o -perm 600 -o -perm 444 \
|
||||
-o -perm 440 -o -perm 400 \) -exec chmod 644 {} \;
|
||||
|
||||
make PREFIX=/usr
|
||||
make install PREFIX=/usr DESTDIR=$PKG
|
||||
|
||||
find $PKG -print0 | xargs -0 file | grep -e "executable" -e "shared object" | grep ELF \
|
||||
| cut -f 1 -d : | xargs strip --strip-unneeded 2> /dev/null || true
|
||||
|
||||
find $PKG/usr/man -type f -exec gzip -9 {} \;
|
||||
for i in $( find $PKG/usr/man -type l ) ; do ln -s $( readlink $i ).gz $i.gz ; rm $i ; done
|
||||
|
||||
mkdir -p $PKG/usr/doc/$PRGNAM-$VERSION
|
||||
cp -a \
|
||||
COPYING CONTRIBUTORS CHANGELOG TODO README \
|
||||
$PKG/usr/doc/$PRGNAM-$VERSION
|
||||
cat $CWD/$PRGNAM.SlackBuild > $PKG/usr/doc/$PRGNAM-$VERSION/$PRGNAM.SlackBuild
|
||||
|
||||
mkdir -p $PKG/install
|
||||
cat $CWD/slack-desc > $PKG/install/slack-desc
|
||||
|
||||
cd $PKG
|
||||
/sbin/makepkg -l y -c n $OUTPUT/$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE
|
|
@ -0,0 +1,10 @@
|
|||
PRGNAM="splitjob"
|
||||
VERSION="3.2"
|
||||
HOMEPAGE="http://splitjob.sourceforge.net/"
|
||||
DOWNLOAD="https://master.dl.sourceforge.net/project/splitjob/3.2stable/splitjob-3.2.tar.xz"
|
||||
MD5SUM="e11d35fced4b34de1ac5196c257d2b20"
|
||||
DOWNLOAD_x86_64=""
|
||||
MD5SUM_x86_64=""
|
||||
REQUIRES=""
|
||||
MAINTAINER="Henrik Carlqvist"
|
||||
EMAIL="hc1363@poolhem.se"
|
Loading…
Reference in New Issue