FAQ Search Today's Posts Mark Forums Read
» Video Reviews

» Linux Archive

Linux-archive is a website aiming to archive linux email lists and to make them easily accessible for linux users/developers.


» Sponsor

» Partners

» Sponsor

Go Back   Linux Archive > ArchLinux > ArchLinux Pacman Development

 
 
LinkBack Thread Tools
 
Old 02-25-2009, 05:43 PM
Xavier Chantry
 
Default repo-add : rework of delta support

Use the correct database format

Instead of getting all the info from the filename, which was a bit ugly, use
xdelta3 to get the source and destination files

Instead of looking for .delta files in the current directory when adding a
package, which was not very flexible, allow .delta files to be added with
repo-add just like package files. delta files can also be removed with
repo-remove. This is simply done by looking for a .delta extension in the
arguments, and calling the appropriate db_write_delta or db_remove_delta
functions.

Example usage:
repo-add repo/test.db.tar.gz repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz
repo-add repo/test.db.tar.gz repo/libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta
repo-remove repo/test.db.tar.gz libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta

Note that db_write_entry had to be modified so that the deltas file is not
lost on a package upgrade (remove + add). FS#13414 should be fixed in the
same time, by printing a different message when the same package is added.

Normal output:
==> Adding package 'repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz'
-> Removing version 'libx11-1.1.5-2'...
Warning:
==> Adding package 'repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz'
==> WARNING: An entry for 'libx11-1.1.99.2-2' already exists, overwriting...

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
---
scripts/repo-add.sh.in | 203 ++++++++++++++++++++++++++++--------------------
1 files changed, 120 insertions(+), 83 deletions(-)

diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in
index c6d25aa..95b7959 100644
--- a/scripts/repo-add.sh.in
+++ b/scripts/repo-add.sh.in
@@ -57,8 +57,8 @@ error() {
# print usage instructions
usage() {
printf "repo-add, repo-remove (pacman) %s

" "$myver"
- printf "$(gettext "Usage: repo-add [-q] <path-to-db> <package> ...
")"
- printf "$(gettext "Usage: repo-remove [-q] <path-to-db> <packagename> ...

")"
+ printf "$(gettext "Usage: repo-add [-q] <path-to-db> <package|delta> ...
")"
+ printf "$(gettext "Usage: repo-remove [-q] <path-to-db> <packagename|delta> ...

")"
printf "$(gettext "
repo-add will update a package database by reading a package file.

Multiple packages to add can be specified on the command line.

")"
@@ -93,36 +93,84 @@ write_list_entry() {
fi
}

-# write a delta entry to the pacman database
-# arg1 - path to delta
+# Get the package name from the delta filename
+getpkgname() {
+ local tmp
+
+ tmp=${1##*/}
+ echo ${tmp%-*-*_to*}
+}
+
+find_pkgentry()
+{
+ local pkgname=$1
+ local pkgentry
+ for pkgentry in $gstmpdir/$pkgname*; do
+ name=${pkgentry##*/}
+ if [ "${name%-*-*}" = "$pkgname" ]; then
+ echo $pkgentry
+ return 0
+ fi
+ done
+ return 1
+}
+
+# write a delta entry
+# arg1 - path to delta file
db_write_delta()
{
- # blank out all variables
- local deltafile="$1"
- local filename=$(basename "$deltafile")
- local deltavars pkgname fromver tover arch csize md5sum
-
- # format of the delta filename:
- # (package)-(fromver)_to_(tover)-(arch).delta
- deltavars=( $(echo "$filename" | sed -e 's/(.*)-(.*-.*)_to_(.*-.*)-(.*).delta/1 2 3 4/') )
- pkgname=${deltavars[0]}
- fromver=${deltavars[1]}
- tover=${deltavars[2]}
- arch=${deltavars[3]}
-
- # get md5sum and size of delta
+ deltafile="$1"
+ pkgname="$(getpkgname $deltafile)"
+
+ pkgentry=$(find_pkgentry $pkgname)
+ if [ -z "$pkgentry" ]; then
+ return 1
+ fi
+ deltas="$pkgentry/deltas"
+ # create deltas file if it does not already exist
+ if [ ! -f "$deltas" ]; then
+ msg2 "$(gettext "Creating 'deltas' db entry...")"
+ echo -e "%DELTAS%" >>$deltas
+ fi
+ # get md5sum and compressed size of package
md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')"
csize=$(@SIZECMD@ "$deltafile")

- # ensure variables were found
- if [ -z "$pkgname" -o -z "$fromver" -o -z "$tover" -o -z "$arch" ]; then
- return 1
+ oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//')
+ newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//')
+
+ if grep -q "$oldfile.*$newfile" $deltas; then
+ warning "$(gettext "An entry for '%s' already exists, overwriting...")" "$deltafile"
+ sed -i "/$oldfile.*$newfile/d" $deltas
fi
+ echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas

- # add the entry for this delta file
- echo -e "$fromver $tover $csize $filename $md5sum" >>deltas
+ return 0
} # end db_write_delta

+# remove a delta entry
+# arg1 - path to delta file
+db_remove_delta()
+{
+ deltafile="$1"
+ filename=${deltafile##*/}
+ pkgname="$(getpkgname $deltafile)"
+
+ pkgentry=$(find_pkgentry $pkgname)
+ if [ -z "$pkgentry" ]; then
+ return 1
+ fi
+ deltas="$pkgentry/deltas"
+ if [ ! -f "$deltas" ]; then
+ return 1
+ fi
+ if grep -q "$filename" $deltas; then
+ sed -i "/$filename/d" $deltas
+ return 0
+ fi
+
+ return 1
+} # end db_write_delta

# write an entry to the pacman database
# arg1 - path to package
@@ -172,19 +220,34 @@ db_write_entry()
return 1
fi

- # remove an existing entry if it exists, ignore failures
- db_remove_entry "$pkgname"
-
startdir=$(pwd)
pushd "$gstmpdir" 2>&1 >/dev/null

- # create package directory
- mkdir "$pkgname-$pkgver"
+ # remove an eventual existing entry but keep the deltas file
+ if [ -d "$pkgname-$pkgver" ]; then
+ # special case : same version already exists
+ # package directory can be kept, just delete depends and desc
+ warning "$(gettext "An entry for '%s' already exists, overwriting...")" "$pkgname-$pkgver"
+ rm "$pkgname-$pkgver"/{depends,desc}
+ else
+ pkgentry=$(find_pkgentry $pkgname)
+ # create package directory
+ mkdir "$pkgname-$pkgver"
+ if [ -n "$pkgentry" ]; then
+ # keep the deltas file then delete the existing entry
+ if [ -f "$pkgentry/deltas" ]; then
+ cp "$pkgentry/deltas" "$pkgname-$pkgver"
+ fi
+ msg2 "$(gettext "Removing version '%s'...")" "${pkgentry##*/}"
+ rm -rf $pkgentry
+ fi
+ fi
+
cd "$pkgname-$pkgver"

# create desc entry
msg2 "$(gettext "Creating 'desc' db entry...")"
- echo -e "%FILENAME%
$(basename "$1")
" >>desc
+ echo -e "%FILENAME%
"${1##*/}"
" >>desc
echo -e "%NAME%
$pkgname
" >>desc
echo -e "%VERSION%
$pkgver
" >>desc
[ -n "$pkgdesc" ] && echo -e "%DESC%
$pkgdesc
" >>desc
@@ -211,61 +274,16 @@ db_write_entry()
write_list_entry "PROVIDES" "$_provides" "depends"
write_list_entry "OPTDEPENDS" "$_optdepends" "depends"

- # create deltas entry if there are delta files
- # Xav : why should deltas be in $startdir?
- for delta in $startdir/$pkgname-*-*_to_*-*-$arch.delta; do
- # This for loop also pulls in all files that start with the current package
- # name and are followed by a -whatever. For instance, running this loop for
- # gcc would also grab gcc-libs. To guard against this, compare the package
- # name of the delta to the current package name.
- local filename=$(basename "$delta")
- local dpkgname="$(echo "$filename" | sed -e 's/(.*)-.*-.*_to_.*-.*-.*.delta/1/')"
- if [ "$pkgname" = "$dpkgname" -a -f "$delta" ]; then
- # create deltas file if it does not already exist
- if [ ! -f "deltas" ]; then
- msg2 "$(gettext "Creating 'deltas' db entry...")"
- echo -e "%DELTAS%" >>deltas
- fi
-
- # write this delta entry
- if db_write_delta "$delta"; then
- msg2 "$(gettext "Added delta '%s'")" "$(basename "$delta")"
- else
- warning "$(gettext "Could not add delta '%s'")" "$(basename "$delta")"
- fi
- fi
- done
- # add the final newline
- [ -f "deltas" ] && echo -e "" >>deltas
-
popd 2>&1 >/dev/null

# preserve the modification time
# Xav : what for?
pkgdir="$gstmpdir/$pkgname-$pkgver"
touch -r "$pkgfile" "$pkgdir/desc" "$pkgdir/depends"
- [ -f "$pkgdir/deltas" ] && touch -r "$pkgfile" "$pkgdir/deltas"

return 0
} # end db_write_entry

-# remove existing entries from the DB
-# arg1 - package name
-db_remove_entry() {
- pushd "$gstmpdir" 2>&1 >/dev/null
-
- # remove any other package in the DB with same name
- local existing
- for existing in *; do
- if [ "${existing%-*-*}" = "$1" ]; then
- msg2 "$(gettext "Removing existing package '%s'...")" "$existing"
- rm -rf "$existing"
- fi
- done
-
- popd 2>&1 >/dev/null
-} # end db_remove_entry
-
# PROGRAM START

# determine whether we have gettext; make it a no-op if we do not
@@ -299,7 +317,7 @@ gstmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (
exit 1)

# figure out what program we are
-cmd="$(basename $0)"
+cmd=${0##*/}
if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then
error "$(gettext "Invalid command name '%s' specified.")" "$cmd"
exit 1
@@ -329,7 +347,14 @@ for arg in "$@"; do
else
if [ "$cmd" == "repo-add" ]; then
if [ -f "$arg" ]; then
- if ! bsdtar -tf "$arg" .PKGINFO 2>&1 >/dev/null; then
+ if [ "${arg##*.}" == "delta" ]; then
+ msg "$(gettext "Adding delta '%s'")" "$arg"
+ if db_write_delta "$arg"; then
+ success=1
+ else
+ error "$(gettext "No package entry for '%s'.")" "$arg"
+ fi
+ elif ! bsdtar -tf "$arg" .PKGINFO 2>&1 >/dev/null; then
error "$(gettext "'%s' is not a package file, skipping")" "$arg"
else
msg "$(gettext "Adding package '%s'")" "$arg"
@@ -339,15 +364,27 @@ for arg in "$@"; do
fi
fi
else
- error "$(gettext "Package '%s' not found.")" "$arg"
+ error "$(gettext "File '%s' not found.")" "$arg"
fi
elif [ "$cmd" == "repo-remove" ]; then
- msg "$(gettext "Searching for package '%s'...")" "$arg"
-
- if db_remove_entry "$arg"; then
- success=1
+ if [ "${arg##*.}" == "delta" ]; then
+ msg "$(gettext "Searching for delta '%s'...")" "$arg"
+ if db_remove_delta "$arg"; then
+ success=1
+ else
+ error "$(gettext "Delta matching '%s' not found.")" "$arg"
+ fi
else
- error "$(gettext "Package matching '%s' not found.")" "$arg"
+ msg "$(gettext "Searching for package '%s'...")" "$arg"
+
+ pkgentry=$(find_pkgentry $arg)
+ if [ -n "$pkgentry" ]; then
+ msg2 "$(gettext "Removing existing version '%s'...")" "${pkgentry##*/}"
+ rm -rf $pkgentry
+ success=1
+ else
+ error "$(gettext "Package matching '%s' not found.")" "$arg"
+ fi
fi
fi
fi
@@ -364,7 +401,7 @@ if [ $success -eq 1 ]; then
"$REPO_DB_FILE" ;;
esac

- filename=$(basename "$REPO_DB_FILE")
+ filename=${REPO_DB_FILE##*/}

pushd "$gstmpdir" 2>&1 >/dev/null
if [ -n "$(ls)" ]; then
--
1.6.1.3

_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 02:11 AM
Allan McRae
 
Default repo-add : rework of delta support

Xavier Chantry wrote:

Use the correct database format

Instead of getting all the info from the filename, which was a bit ugly, use
xdelta3 to get the source and destination files

Instead of looking for .delta files in the current directory when adding a
package, which was not very flexible, allow .delta files to be added with
repo-add just like package files. delta files can also be removed with
repo-remove. This is simply done by looking for a .delta extension in the
arguments, and calling the appropriate db_write_delta or db_remove_delta
functions.

Example usage:
repo-add repo/test.db.tar.gz repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz
repo-add repo/test.db.tar.gz repo/libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta
repo-remove repo/test.db.tar.gz libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta

Note that db_write_entry had to be modified so that the deltas file is not
lost on a package upgrade (remove + add). FS#13414 should be fixed in the
same time, by printing a different message when the same package is added.

Normal output:
==> Adding package 'repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz'
-> Removing version 'libx11-1.1.5-2'...
Warning:
==> Adding package 'repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz'
==> WARNING: An entry for 'libx11-1.1.99.2-2' already exists, overwriting...

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
---
scripts/repo-add.sh.in | 203 ++++++++++++++++++++++++++++--------------------
1 files changed, 120 insertions(+), 83 deletions(-)

diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in
index c6d25aa..95b7959 100644
--- a/scripts/repo-add.sh.in
+++ b/scripts/repo-add.sh.in
@@ -57,8 +57,8 @@ error() {
# print usage instructions
usage() {
printf "repo-add, repo-remove (pacman) %s

" "$myver"
- printf "$(gettext "Usage: repo-add [-q] <path-to-db> <package> ...
")"
- printf "$(gettext "Usage: repo-remove [-q] <path-to-db> <packagename> ...

")"
+ printf "$(gettext "Usage: repo-add [-q] <path-to-db> <package|delta> ...
")"
+ printf "$(gettext "Usage: repo-remove [-q] <path-to-db> <packagename|delta> ...

")"
printf "$(gettext "
repo-add will update a package database by reading a package file.

Multiple packages to add can be specified on the command line.

")"
@@ -93,36 +93,84 @@ write_list_entry() {
fi
}

-# write a delta entry to the pacman database

-# arg1 - path to delta
+# Get the package name from the delta filename
+getpkgname() {
+ local tmp
+
+ tmp=${1##*/}
+ echo ${tmp%-*-*_to*}
+}



Rename this to getdeltapkgname to clarify it is only used for deltas?
Self document function names make my head hurt less.


Otherwise, a quick read of the other changes made sense to me, but I
will need to do this in more detail.

One comment that is moderately related to this. Should there be a
mechanism to inspect the deltas are remove the ones that are no longer
necessary? I.e. deltas with a broken chain, deltas who chain download
will be bigger than the package. Should that be a job of repo-add when
adding a new delta or a separate script?


Allan



_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 07:01 AM
Xavier
 
Default repo-add : rework of delta support

On Thu, Feb 26, 2009 at 4:11 AM, Allan McRae <allan@archlinux.org> wrote:
>> +# Get the package name from the delta filename
>> +getpkgname() {
>> + * * * local tmp
>> +
>> + * * * tmp=${1##*/}
>> + * * * echo ${tmp%-*-*_to*}
>> +}
>>
>
> Rename this to getdeltapkgname to clarify it is only used for deltas? *Self
> document function names make my head hurt less.
>

Sorry, I had exactly the same thought, I just didn't know where to add
delta I will use your suggestion.

> Otherwise, a quick read of the other changes made sense to me, but I will
> need to do this in more detail.

This is probably the patch that requires the most careful review
(compared to the other 4), so it would be highly appreciated.

> One comment that is moderately related to this. *Should there be a mechanism
> to inspect the deltas are remove the ones that are no longer necessary?
> *I.e. deltas with a broken chain, deltas who chain download will be bigger
> than the package. *Should that be a job of repo-add when adding a new delta
> or a separate script?
>

I don't know, that seems neat but maybe too much complexity, I am not
sure. Finding out how deltas could/should be managed has always been a
big issue to me, how/when they are added, how/when they are removed.
_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 07:58 AM
Brendan Hide
 
Default repo-add : rework of delta support

Xavier wrote:

On Thu, Feb 26, 2009 at 4:11 AM, Allan McRae <allan@archlinux.org> wrote:


One comment that is moderately related to this. Should there be a mechanism
to inspect the deltas are remove the ones that are no longer necessary?
I.e. deltas with a broken chain, deltas who chain download will be bigger
than the package. Should that be a job of repo-add when adding a new delta
or a separate script?


I don't know, that seems neat but maybe too much complexity, I am not
sure. Finding out how deltas could/should be managed has always been a
big issue to me, how/when they are added, how/when they are removed.


Hi Guys

I was just getting into understanding the code to see what I could do to
get this going myself. Thank you, Xavier - I think it would have taken
me a week or 2 to put all this together and likely not nearly good
enough for Allan to be saying mostly good things about it.


Now that you mentioned it, other than broken chains and deltas bigger
than "deltaless" downloads, what other criteria are there for delta
removal? I can only think of disk space - though this is not likely to
be an issue. Also on that note, other than this new code breaking the
chain, how would a chain be broken?

_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 08:25 AM
Allan McRae
 
Default repo-add : rework of delta support

Xavier wrote:

On Thu, Feb 26, 2009 at 4:11 AM, Allan McRae <allan@archlinux.org> wrote:


+# Get the package name from the delta filename
+getpkgname() {
+ local tmp
+
+ tmp=${1##*/}
+ echo ${tmp%-*-*_to*}
+}



Rename this to getdeltapkgname to clarify it is only used for deltas? Self
document function names make my head hurt less.




Sorry, I had exactly the same thought, I just didn't know where to add
delta I will use your suggestion.



Otherwise, a quick read of the other changes made sense to me, but I will
need to do this in more detail.



This is probably the patch that requires the most careful review
(compared to the other 4), so it would be highly appreciated.



OK. I will pull your patches into a git branch to remind myself to go
through these in a lot of detail sometime in the next week or so. Do
you have a git home now that Dan moved his server or do I need to apply
them from the mailing list? shining.toofishes.net does not seem to
exist anymore.


Did you test whether creating a repo db from (e.g.) your cache gave the
same result with the new and old version. I don't remember anything that
should change just adding packages.



One comment that is moderately related to this. Should there be a mechanism
to inspect the deltas are remove the ones that are no longer necessary?
I.e. deltas with a broken chain, deltas who chain download will be bigger
than the package. Should that be a job of repo-add when adding a new delta
or a separate script?




I don't know, that seems neat but maybe too much complexity, I am not
sure. Finding out how deltas could/should be managed has always been a
big issue to me, how/when they are added, how/when they are removed.


I think deciding when they should be removed is the easiest part here:
1) when you can not get from a delta to the current package, it should
be removed
2) when getting from a delta to the current package requires more
download than just downloading the current package (or whatever
criterion pacman uses - 80%?), it should be deleted. Perhaps this
should be a separate script, or maybe a flag (repo-add --cleandelta)?


Deciding when to add them is very political...

Allan



_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 08:31 AM
Allan McRae
 
Default repo-add : rework of delta support

Brendan Hide wrote:
Also on that note, other than this new code breaking the chain, how
would a chain be broken?


By someone choosing not to or forgetting to add a delta. At the moment
repo-add does not create deltas automatically, or at all... If it ever
did create deltas, it would have to be through the use of a flag. Same
with file list generation (http://bugs.archlinux.org/task/11302). Some
people don't want their repo server dealing with the overhead. (I had
to add >150 packages to the repo in one go for an ncurses rebuild).


Allan



_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 07:05 PM
Xavier
 
Default repo-add : rework of delta support

On Thu, Feb 26, 2009 at 10:25 AM, Allan McRae <allan@archlinux.org> wrote:
> Xavier wrote:
>>
>> This is probably the patch that requires the most careful review
>> (compared to the other 4), so it would be highly appreciated.
>>
>
> OK. *I will pull your patches into a git branch to remind myself to go
> through these in a lot of detail sometime in the next week or so. *Do you
> have a git home now that Dan moved his server or do I need to apply them
> from the mailing list? *shining.toofishes.net does not seem to exist
> anymore.
>

I reworked and splitted this big repo-add patch in 4 patches :
0001-repo-add-print-warning-if-same-version-already-exis.patch
0004-repo-add.sh.in-repo-remove-improvements.patch
0005-repo-add-drop-delta-support-to-rewrite-it-from-scr.patch
0006-repo-add-rewrite-delta-support.patch

I also added two patches :
0002-repo-add-fail-early-if-repo-can-not-be-created.patch
0003-repo-add-cleanup.patch

These 6 patches are on my repoadd branch, I will send them to the list
for review
_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 10:00 PM
Allan McRae
 
Default repo-add : rework of delta support

Xavier wrote:

On Thu, Feb 26, 2009 at 10:25 AM, Allan McRae <allan@archlinux.org> wrote:


Xavier wrote:


This is probably the patch that requires the most careful review
(compared to the other 4), so it would be highly appreciated.



OK. I will pull your patches into a git branch to remind myself to go
through these in a lot of detail sometime in the next week or so. Do you
have a git home now that Dan moved his server or do I need to apply them
from the mailing list? shining.toofishes.net does not seem to exist
anymore.




I reworked and splitted this big repo-add patch in 4 patches :
0001-repo-add-print-warning-if-same-version-already-exis.patch
0004-repo-add.sh.in-repo-remove-improvements.patch
0005-repo-add-drop-delta-support-to-rewrite-it-from-scr.patch
0006-repo-add-rewrite-delta-support.patch

I also added two patches :
0002-repo-add-fail-early-if-repo-can-not-be-created.patch
0003-repo-add-cleanup.patch

These 6 patches are on my repoadd branch, I will send them to the list
for review



You could have branched the repoadd branch from your working branch so I
could get all patches in one go. But combining the two has improved my
git-fu so I graciously accept the lesson you were trying to provide
me...


Anyway, your patches are a lot easier for me to digest at this size. I
will post any comments later.


Allan



_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 02-26-2009, 10:08 PM
Xavier
 
Default repo-add : rework of delta support

On Fri, Feb 27, 2009 at 12:00 AM, Allan McRae <allan@archlinux.org> wrote:
>
> You could have branched the repoadd branch from your working branch so I
> could get all patches in one go. But combining the two has improved my
> git-fu so I graciously accept the lesson you were trying to provide me...
> *
>

Hm indeed, it was useful to have a second branch temporarily, but when
it was done, I could have merged everything together.
Actually I did for my own use, so that I could test everything together :P
I guess I will put back everything on the working branch tomorrow.

> Anyway, your patches are a lot easier for me to digest at this size. *I will
> post any comments later.
>

Cool I am also much happier about them.
_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 
Old 03-01-2009, 06:34 PM
Xavier
 
Default repo-add : rework of delta support

On Thu, Feb 26, 2009 at 10:25 AM, Allan McRae <allan@archlinux.org> wrote:
>
> I think deciding when they should be removed is the easiest part here:
> 1) when you can not get from a delta to the current package, it should be
> removed
> 2) when getting from a delta to the current package requires more download
> than just downloading the current package (or whatever criterion pacman uses
> - 80%?), it should be deleted. *Perhaps this should be a separate script, or
> maybe a flag (repo-add --cleandelta)?
>

What is a bit disappointing is that this means reimplementing the
logic we already have in pacman, but well.
Btw, the delta sizes could not be more variable. The full range is
obtained, just using my not so big cache : 0% - 100% (of the newest
package)

I stole and hacked a script from the forums to generate the numbers I wanted :
http://bbs.archlinux.org/viewtopic.php?pid=433730#p433730
Thanks sabooky

0% : ghostscript-8.64-2_to_8.64-3-x86_64.delta
0% : ttf-bitstream-vera-1.10-5_to_1.10-6-x86_64.delta
0% : sound-theme-freedesktop-0.1-1_to_0.2-1-x86_64.delta
0% : libmad-0.15.1b-3_to_0.15.1b-4-x86_64.delta
0% : tdb-3.3.0-1_to_3.3.1-1-x86_64.delta
1% : libvorbis-1.2.1rc1-1_to_1.2.1rc1-2-x86_64.delta
2% : gnome-power-manager-2.24.2-2_to_2.24.4-1-x86_64.delta
2% : teeworlds-0.5.1-1_to_0.5.1-2-x86_64.delta
3% : ghostscript-8.64-1_to_8.64-2-x86_64.delta
3% : xextproto-7.0.4-1_to_7.0.5-1-x86_64.delta
4% : openjdk6-1.4-2_to_1.4.1-1-x86_64.delta
4% : xcb-proto-1.3-1_to_1.4-1-x86_64.delta
7% : tzdata-2009b-1_to_2009a-1-x86_64.delta
7% : alsa-utils-1.0.18-1_to_1.0.19-1-x86_64.delta
7% : groff-1.20.1-1_to_1.20.1-2-x86_64.delta
8% : xcb-util-0.3.2-1_to_0.3.3-1-x86_64.delta
8% : firefox-3.0.5-1_to_3.0.6-1-x86_64.delta
8% : pm-utils-1.2.3-4_to_1.2.4-1-x86_64.delta
9% : kernel26-2.6.28.3-1_to_2.6.28.4-1-x86_64.delta
9% : kernel26-2.6.28.5-1_to_2.6.28.6-1-x86_64.delta
9% : kernel26-2.6.28.6-1_to_2.6.28.7-1-x86_64.delta
10% : libxcb-1.1.93-1_to_1.2-1-x86_64.delta
10% : libxi-1.1.4-1_to_1.1.4-2-x86_64.delta
11% : kernel26-2.6.28.4-1_to_2.6.28.5-1-x86_64.delta
12% : xkeyboard-config-1.4-2_to_1.5-1-x86_64.delta
13% : xcb-proto-1.2-2_to_1.3-1-x86_64.delta
14% : libdrm-2.3.1-2_to_2.3.1-3-x86_64.delta
16% : gimp-2.6.4-2_to_2.6.5-1-x86_64.delta
16% : inputproto-1.4.4-1_to_1.5.0-1-x86_64.delta
18% : bluez-4.29-1_to_4.30-1-x86_64.delta
20% : evolution-data-server-2.24.3-1_to_2.24.4.1-1-x86_64.delta
23% : libxml2-2.7.3-1.1_to_2.7.2-1-x86_64.delta
24% : pciutils-3.0.3-1_to_3.1.2-1-x86_64.delta
27% : ntfs-3g-2009.1.1-1_to_2009.2.1-1-x86_64.delta
27% : flashplugin-10.0.d21.1-1_to_10.0.22.87-1-x86_64.delta
29% : libv4l-0.5.7-1_to_0.5.8-1-x86_64.delta
29% : gnutls-2.6.3-1_to_2.6.4-1-x86_64.delta
29% : man-pages-3.17-1_to_3.18-1-x86_64.delta
31% : ruby-1.8.7_p72-2_to_1.8.7_p72-3-x86_64.delta
33% : device-mapper-1.02.29-1_to_1.02.30-1-x86_64.delta
34% : libpng-1.2.34-1_to_1.2.35-1-x86_64.delta
37% : libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta
38% : gpm-1.20.5-2_to_1.20.6-1-x86_64.delta
40% : dhcpcd-4.0.7-1_to_4.0.10-1-x86_64.delta
41% : libx11-1.1.99.2-2_to_1.2-1-x86_64.delta
41% : xorg-xinit-1.1.0-1_to_1.1.1-1-x86_64.delta
42% : pixman-0.12.0-1_to_0.14.0-1-x86_64.delta
42% : hal-0.5.11-4_to_0.5.11-7-x86_64.delta
42% : tdb-3.2.7-1_to_3.3.0-1-x86_64.delta
46% : libxfont-1.3.4-1_to_1.4.0-1-x86_64.delta
46% : gsynaptics-0.9.14-2_to_0.9.15-1-x86_64.delta
49% : libmad-0.15.1b-2_to_0.15.1b-3-x86_64.delta
49% : xf86-input-evdev-2.1.0-1_to_2.1.2-1-x86_64.delta
49% : whois-4.7.28-2_to_4.7.30-1-x86_64.delta
49% : sqlite3-3.6.10-1_to_3.6.11-1-x86_64.delta
50% : gnome-mplayer-0.9.3-1_to_0.9.4-1-x86_64.delta
50% : perl-error-0.17011-1_to_0.17015-1-x86_64.delta
51% : xf86-input-synaptics-0.99.3-1_to_1.0.0-1-x86_64.delta
53% : man-db-2.5.2-2_to_2.5.4-1-x86_64.delta
54% : libxcb-1.1.90.1-1_to_1.1.93-1-x86_64.delta
56% : alsa-lib-1.0.18-1_to_1.0.19-1-x86_64.delta
56% : xf86-input-keyboard-1.3.1-1_to_1.3.2-1-x86_64.delta
58% : libtasn1-1.7-1_to_1.8-1-x86_64.delta
58% : lvm2-2.02.43-1_to_2.02.44-1-x86_64.delta
58% : xterm-239-1_to_241-1-x86_64.delta
59% : xfsprogs-2.10.2-1_to_3.0.0-1-x86_64.delta
60% : dnsutils-9.5.0.P2-1_to_9.6.0.P1-1-x86_64.delta
62% : libxi-1.1.4-2_to_1.2.0-1-x86_64.delta
65% : libxslt-1.1.24-1_to_1.1.24-2-x86_64.delta
65% : hdparm-9.6-1_to_9.10-1-x86_64.delta
67% : libcanberra-0.10-1_to_0.11-2-x86_64.delta
69% : libvorbis-1.2.1rc1-2_to_1.2.0-1-x86_64.delta
73% : smbclient-3.3.0-1_to_3.3.1-1-x86_64.delta
80% : grep-2.5.3-3_to_2.5.4-1-x86_64.delta
83% : smbclient-3.2.7-1_to_3.3.0-1-x86_64.delta
86% : libxext-1.0.4-1_to_1.0.5-1-x86_64.delta
88% : libice-1.0.4-1_to_1.0.5-1-x86_64.delta
89% : faac-1.26-1_to_1.28-1-x86_64.delta
96% : xulrunner-1.9.0.5-2_to_1.9.0.6-1-x86_64.delta
96% : gtk-engine-murrine-0.53.1-1_to_0.53.1-3-x86_64.delta
98% : libcroco-0.6.1-1_to_0.6.2-1-x86_64.delta
100% : pacman-git-20090226-1_to_20090227-2-x86_64.delta
101% : cabextract-1.2-1_to_1.2-2-x86_64.delta

That's right, 101%, no error here. The delta is 32427 B while the
package is 32045 B

Since pacman indeed has a max ratio (#define MAX_DELTA_RATIO 0.7), it
doesn't make any sense to put deltas bigger than 70% in a database,
because pacman will never use them.
Do we enforce this check somewhere? If yes where? In pkgdelta or in
repo-add and/or in an external cleanup script?
_______________________________________________
pacman-dev mailing list
pacman-dev@archlinux.org
http://www.archlinux.org/mailman/listinfo/pacman-dev
 

Thread Tools




All times are GMT. The time now is 05:36 AM.

VBulletin, Copyright ©2000 - 2014, Jelsoft Enterprises Ltd.
Content Relevant URLs by vBSEO ©2007, Crawlability, Inc.
Copyright 2007 - 2008, www.linux-archive.org