This patch adds in support for automatically detecting and decompressing
both formats gentoo currently ships portage snapshots in. To make the
change as minimal as possible I am using recent gnu tar ability to
automatically pick the correct format and decompress. This may not work
as expected on *bsd or MacOSX unless we already require gnu tar on those
platforms.

Signed-off-by: Rick Farina <sidhayn@gmail.com>
---
modules/generic_stage_target.py | 13 ++++++++-----
1 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py
index c4ef239..031d8bf 100644
--- a/modules/generic_stage_target.py
+++ b/modules/generic_stage_target.py
@@ -404,12 +404,15 @@ class generic_stage_target(generic_target):

def set_snapshot_path(self):
self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
- "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
+ "/snapshots/portage-"+self.settings["snapshot"]+".tar.xz")

if os.path.exists(self.settings["snapshot_path"]):
self.settings["snapshot_path_hash"]=
generate_hash(self.settings["snapshot_path"],
hash_function=self.settings["hash_function"],verbose=False)
+ else:
+ self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
+ "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")

def set_snapcache_path(self):
if self.settings.has_key("SNAPCACHE"):
@@ -635,7 +638,7 @@ class generic_stage_target(generic_target):
self.settings["source_path"]+"
to "+
self.settings["chroot_path"]+
" (This may take some time) ...
"
- unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+
+ unpack_cmd="tar xpf "+self.settings["source_path"]+" -C "+
self.settings["chroot_path"]
error_msg="Tarball extraction of "+
self.settings["source_path"]+" to "+
@@ -646,7 +649,7 @@ class generic_stage_target(generic_target):
self.settings["source_path"]+"
to "+
self.settings["chroot_path"]+
" (This may take some time) ...
"
- unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+
+ unpack_cmd="tar xpf "+self.settings["source_path"]+" -C "+
self.settings["chroot_path"]
error_msg="Tarball extraction of "+self.settings["source_path"]+
" to "+self.settings["chroot_path"]+" failed."
@@ -744,7 +747,7 @@ class generic_stage_target(generic_target):
read_from_clst(self.settings["snapshot_cache_path"]+
"catalyst-hash")
destdir=self.settings["snapshot_cache_path"]
- unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
+ unpack_cmd="tar xpf "+self.settings["snapshot_path"]+" -C "+destdir
unpack_errmsg="Error unpacking snapshot"
cleanup_msg="Cleaning up invalid snapshot cache at
"+
self.settings["snapshot_cache_path"]+
@@ -760,7 +763,7 @@ class generic_stage_target(generic_target):
cleanup_errmsg="Error removing existing snapshot directory."
cleanup_msg=
"Cleaning up existing portage tree (This can take a long time)..."
- unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+
+ unpack_cmd="tar xpf "+self.settings["snapshot_path"]+" -C "+
self.settings["chroot_path"]+"/usr"
unpack_errmsg="Error unpacking snapshot"

--
1.7.8.6

Is the gnu tar requirement too much? I can rewrite it without but this
way is just so clean...

Thanks,
Zero

On 08/21/2012 08:58 PM, Rick Farina wrote:
> This patch adds in support for automatically detecting and decompressing
> both formats gentoo currently ships portage snapshots in. To make the
> change as minimal as possible I am using recent gnu tar ability to
> automatically pick the correct format and decompress. This may not work
> as expected on *bsd or MacOSX unless we already require gnu tar on those
> platforms.
>
> Signed-off-by: Rick Farina <sidhayn@gmail.com>
> ---
> modules/generic_stage_target.py | 13 ++++++++-----
> 1 files changed, 8 insertions(+), 5 deletions(-)
>
> diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py
> index c4ef239..031d8bf 100644
> --- a/modules/generic_stage_target.py
> +++ b/modules/generic_stage_target.py
> @@ -404,12 +404,15 @@ class generic_stage_target(generic_target):
>
> def set_snapshot_path(self):
> self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
> - "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
> + "/snapshots/portage-"+self.settings["snapshot"]+".tar.xz")
>
> if os.path.exists(self.settings["snapshot_path"]):
> self.settings["snapshot_path_hash"]=
> generate_hash(self.settings["snapshot_path"],
> hash_function=self.settings["hash_function"],verbose=False)
> + else:
> + self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
> + "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
>
> def set_snapcache_path(self):
> if self.settings.has_key("SNAPCACHE"):
> @@ -635,7 +638,7 @@ class generic_stage_target(generic_target):
> self.settings["source_path"]+"
to "+
> self.settings["chroot_path"]+
> " (This may take some time) ...
"
> - unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+
> + unpack_cmd="tar xpf "+self.settings["source_path"]+" -C "+
> self.settings["chroot_path"]
> error_msg="Tarball extraction of "+
> self.settings["source_path"]+" to "+
> @@ -646,7 +649,7 @@ class generic_stage_target(generic_target):
> self.settings["source_path"]+"
to "+
> self.settings["chroot_path"]+
> " (This may take some time) ...
"
> - unpack_cmd="tar xjpf "+self.settings["source_path"]+" -C "+
> + unpack_cmd="tar xpf "+self.settings["source_path"]+" -C "+
> self.settings["chroot_path"]
> error_msg="Tarball extraction of "+self.settings["source_path"]+
> " to "+self.settings["chroot_path"]+" failed."
> @@ -744,7 +747,7 @@ class generic_stage_target(generic_target):
> read_from_clst(self.settings["snapshot_cache_path"]+
> "catalyst-hash")
> destdir=self.settings["snapshot_cache_path"]
> - unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+destdir
> + unpack_cmd="tar xpf "+self.settings["snapshot_path"]+" -C "+destdir
> unpack_errmsg="Error unpacking snapshot"
> cleanup_msg="Cleaning up invalid snapshot cache at
"+
> self.settings["snapshot_cache_path"]+
> @@ -760,7 +763,7 @@ class generic_stage_target(generic_target):
> cleanup_errmsg="Error removing existing snapshot directory."
> cleanup_msg=
> "Cleaning up existing portage tree (This can take a long time)..."
> - unpack_cmd="tar xjpf "+self.settings["snapshot_path"]+" -C "+
> + unpack_cmd="tar xpf "+self.settings["snapshot_path"]+" -C "+
> self.settings["chroot_path"]+"/usr"
> unpack_errmsg="Error unpacking snapshot"
>
>

On Tue, Aug 21, 2012 at 5:58 PM, Rick Farina <sidhayn@gmail.com> wrote:
> This patch adds in support for automatically detecting and decompressing
> both formats gentoo currently ships portage snapshots in. To make the
> change as minimal as possible I am using recent gnu tar ability to
> automatically pick the correct format and decompress. This may not work
> as expected on *bsd or MacOSX unless we already require gnu tar on those
> platforms.
>
> Signed-off-by: Rick Farina <sidhayn@gmail.com>
> ---
> modules/generic_stage_target.py | 13 ++++++++-----
> 1 files changed, 8 insertions(+), 5 deletions(-)
>
> diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py
> index c4ef239..031d8bf 100644
> --- a/modules/generic_stage_target.py
> +++ b/modules/generic_stage_target.py
> @@ -404,12 +404,15 @@ class generic_stage_target(generic_target):
>
> def set_snapshot_path(self):
> self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
> - "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
> + "/snapshots/portage-"+self.settings["snapshot"]+".tar.xz")
>
> if os.path.exists(self.settings["snapshot_path"]):
> self.settings["snapshot_path_hash"]=
> generate_hash(self.settings["snapshot_path"],
> hash_function=self.settings["hash_function"],verbose=False)
> + else:
> + self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
> + "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
>
> def set_snapcache_path(self):
> if self.settings.has_key("SNAPCACHE"):

This bit looks for *.tar.xz and then falls back to *.tar.bz2?

It looks like you don't check that the *.tar.bz2 file exists or that
the hashing is done.

On 08/21/2012 09:05 PM, Matt Turner wrote:
> On Tue, Aug 21, 2012 at 5:58 PM, Rick Farina <sidhayn@gmail.com> wrote:
>> This patch adds in support for automatically detecting and decompressing
>> both formats gentoo currently ships portage snapshots in. To make the
>> change as minimal as possible I am using recent gnu tar ability to
>> automatically pick the correct format and decompress. This may not work
>> as expected on *bsd or MacOSX unless we already require gnu tar on those
>> platforms.
>>
>> Signed-off-by: Rick Farina <sidhayn@gmail.com>
>> ---
>> modules/generic_stage_target.py | 13 ++++++++-----
>> 1 files changed, 8 insertions(+), 5 deletions(-)
>>
>> diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py
>> index c4ef239..031d8bf 100644
>> --- a/modules/generic_stage_target.py
>> +++ b/modules/generic_stage_target.py
>> @@ -404,12 +404,15 @@ class generic_stage_target(generic_target):
>>
>> def set_snapshot_path(self):
>> self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
>> - "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
>> + "/snapshots/portage-"+self.settings["snapshot"]+".tar.xz")
>>
>> if os.path.exists(self.settings["snapshot_path"]):
>> self.settings["snapshot_path_hash"]=
>> generate_hash(self.settings["snapshot_path"],
>> hash_function=self.settings["hash_function"],verbose=False)
>> + else:
>> + self.settings["snapshot_path"]=normpath(self.settings["storedir"]+
>> + "/snapshots/portage-"+self.settings["snapshot"]+".tar.bz2")
>>
>> def set_snapcache_path(self):
>> if self.settings.has_key("SNAPCACHE"):
>
> This bit looks for *.tar.xz and then falls back to *.tar.bz2?
yes, .xz is smaller and faster to decompress hence first.
>
> It looks like you don't check that the *.tar.bz2 file exists or that
> the hashing is done.
>
I forgot to copy and repeat that block, easy fix. I tested and it
didn't appear needed for cleanly failing, but I completely miss the
hash_function and that obviously is bad.

I'll respin on the next flight.

Any comments on the gnu tar requirement? is that an issue or is okay?

thanks,
Zero

On Tue, Aug 21, 2012 at 6:38 PM, Richard Farina <sidhayn@gmail.com> wrote:
> Any comments on the gnu tar requirement? is that an issue or is okay?

I don't really know. The only change required is the addition of the
-J/-z options for non-GNU tar? Or does non-GNU tar support xz?
Wouldn't surprise me if it didn't.

On 08/22/2012 12:27 PM, Matt Turner wrote:
> On Tue, Aug 21, 2012 at 6:38 PM, Richard Farina <sidhayn@gmail.com> wrote:
>> Any comments on the gnu tar requirement? is that an issue or is okay?
>
> I don't really know. The only change required is the addition of the
> -J/-z options for non-GNU tar? Or does non-GNU tar support xz?
> Wouldn't surprise me if it didn't.
>
both gnu and bsd tar work properly the way I have written it. MacOSX tar
does not, but prefix installs gnu tar during bootstrap so it shouldn't
be an issue.