Linux Archive

Linux Archive (http://www.linux-archive.org/)
-   Fedora Build System (http://www.linux-archive.org/fedora-build-system/)
-   -   Perforce support for koji (http://www.linux-archive.org/fedora-build-system/156386-perforce-support-koji.html)

Paul B Schroeder 09-08-2008 09:32 PM

Perforce support for koji
 
Hmm.. I'm looking a bit closer and it looks like I need to fix the
scm_tuple part..


Paul B Schroeder wrote:

The powers that be require us to use Perforce. Thus, the patch. A few
things to note about it:

* Our P4 server is running on a non-standard port and thus needs to be
specified. scm_tuple is now created via rsplit instead. Allows SCM
port to be specified. i.e. This now works:
allowed_scms=perforcetx.bluecoat.com:1999:/koji/sandbox/paul.schroeder/skeletor
* P4 *requires* a stinkin' password. So I added password support to
_parse_url. The "user" part of the SCM url can now be "user:password".
Still works minus the ":password" too. Let me know if there's a
better/preferred way to support passwords.
* P4 can only checkout into the static client root specified in the P4
client spec (PITA). Sooo.. After the module_checkout_cmd is executed,
if sourcedir does not exist, we do a 'os.rename' of the checked out code
into the proper task sourcedir.
* A P4 SCM URL looks like:
p4://user:pass@p4port/p4client/p4depot?path/to/module#changelist
i.e. p4port==host And p4client is your P4 client spec that you've
created.

Let me know if there are any Qs, comments, or any changes I should
make..

Cheers...Paul...


diff --git a/builder/kojid b/builder/kojid
index 1a7afbf..92f8143 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -1830,7 +1830,8 @@ class BuildSRPMFromSCMTask(BaseTaskHandler):
use_common = True

for allowed_scm in options.allowed_scms.split():

- scm_tuple = allowed_scm.split(':')
+ # Use rsplit with 1 max split to allow for port numbers w/
scm host
+ scm_tuple = allowed_scm.rsplit(':', 1)
if len(scm_tuple) in (2, 3):
if fnmatch(scm.host, scm_tuple[0]) and
fnmatch(scm.repository, scm_tuple[1]):
# SCM host:repository is in the allowed list
@@ -2325,7 +2326,8 @@ class SCM(object):
'GIT': ('git://', 'git+http://', 'git+https://', 'git
+rsync://'),
'GIT+SSH': ('git+ssh://',),
'SVN': ('svn://', 'svn+http://', 'svn+https://'),
- 'SVN+SSH': ('svn+ssh://',) }
+ 'SVN+SSH': ('svn+ssh://',),
+ 'P4': ('p4://',) }

def is_scm_url(url):

"""
@@ -2362,10 +2364,11 @@ class SCM(object):
raise koji.GenericError, 'Invalid SCM URL: %s' % url

self.url = url

- scheme, user, host, path, query, fragment = self._parse_url()
+ scheme, user, password, host, path, query, fragment =
self._parse_url()

self.scheme = scheme

self.user = user
+ self.password = password
self.host = host
self.repository = path
self.module = query
@@ -2384,9 +2387,9 @@ class SCM(object):
Parse the SCM url into usable components.
Return the following tuple:

- (scheme, user, host, path, query, fragment)

+ (scheme, user, password, host, path, query, fragment)

- user may be None, everything else will have a value

+ user and password may be None, everything else will have a
value
"""
# get the url's scheme
scheme = self.url.split('://')[0] + '://'
@@ -2396,14 +2399,21 @@ class SCM(object):
dummyscheme, netloc, path, params, query, fragment =
urlparse.urlparse(dummyurl)

user = None

+ password = None
userhost = netloc.split('@')
if len(userhost) == 2:
- user = userhost[0]
+ userpass = userhost[0].split(':')
+ user = userpass[0]
+ if len(userpass) == 2:
+ password = userpass[1]
+ elif len(userpass) > 2:
+ raise koji.GenericError, 'Invalid username:password
specified: %s' % netloc
if not user:
# Don't return an empty string
user = None
- elif ':' in user:
- raise koji.GenericError, 'username:password format not
supported: %s' % user
+ if not password:
+ # Don't return an empty string
+ password = None
netloc = userhost[1]
elif len(userhost) > 2:
raise koji.GenericError, 'Invalid username@hostname
specified: %s' % netloc
@@ -2419,7 +2429,7 @@ class SCM(object):
raise koji.GenericError, 'Unable to parse SCM URL: %s' %
self.url

# return parsed values

- return (scheme, user, netloc, path, query, fragment)
+ return (scheme, user, password, netloc, path, query, fragment)

def checkout(self, scmdir, uploadpath, logfile, use_common=False):

"""
@@ -2514,6 +2524,39 @@ class SCM(object):
module_checkout_cmd = ['svn', 'checkout', '-r',
self.revision, '%s/%s' % (svnserver, self.module), self.module]
common_checkout_cmd = ['svn', 'checkout', '%s/common' %
svnserver]

+ elif self.scmtype == 'P4':

+ if not self.user:
+ raise koji.BuildError, 'No user specified for
repository access scheme: %s' % self.scheme
+ # P4 URL:
+ #
p4://user:pass@p4port/p4client/p4depot?path/to/module#changelist
+
+ # Perforce uses a "client workspace"
+ # Before first slash is the p4 client, after is the p4
depot
+ (p4client, p4depot) = self.repository[1:].split('/', 1)
+
+ p4client_root = None
+ p4info = os.popen('p4 -p ' + self.host + ' -c ' + p4client
+ ' info')
+ for line in p4info:
+ if line.startswith('Client root:'):
+ p4client_root = line.split()[-1]
+ break
+ p4info.close()
+ if not p4client_root:
+ raise koji.BuildError, 'Could not find perforce client
root'
+
+ # Perforce cannot checkout to a random dir (in this case
scmdir).
+ # It will only checkout to the p4client_root, so we
override scmdir
+ # with the p4client_root
+ scmdir = '%s/%s' % (p4client_root, p4depot)
+
+ # When HEAD is given as the revision, we assume we want the
latest
+ p4revision = '
+ if self.revision.upper() != 'HEAD':
+ p4revision = '@%s' % self.revision
+
+ module_checkout_cmd = ['p4', '-p', self.host, '-u',
self.user, '-P', self.password, '-c', p4client, 'sync', '-f', '//%s/%
s/...%s' % (p4depot, self.module, p4revision)]
+ common_checkout_cmd = ['p4', '-p', self.host, '-u',
self.user, '-P', self.password, '-c', p4client, 'sync', '-f', '//%
s/common/...' % p4depot]
+
else:
raise koji.BuildError, 'Unknown SCM type: %s' %
self.scmtype

@@ -2521,6 +2564,10 @@ class SCM(object):

if log_output(module_checkout_cmd[0], module_checkout_cmd,
logfile, uploadpath, cwd=scmdir, logerror=1, env=env):
raise koji.BuildError, 'Error running %s checkout command
"%s", see %s for details' %
(self.scmtype, ' '.join(module_checkout_cmd),
os.path.basename(logfile))
+ # Perforce checks out to the client root listed for the
perforce client.
+ # Need to move checked out source into sourcedir.
+ if not os.path.exists(sourcedir):
+ os.rename('%s/%s' % (scmdir, self.module), sourcedir)

if update_checkout_cmd:

# Currently only required for GIT checkouts


--
Fedora-buildsys-list mailing list
Fedora-buildsys-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-buildsys-list


--
---
Paul B Schroeder <paul.schroeder "at" bluecoat "dot" com>
Blue Coat Systems, Inc.

--
Fedora-buildsys-list mailing list
Fedora-buildsys-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-buildsys-list

Dan Williams 09-08-2008 09:33 PM

Perforce support for koji
 
On Mon, 2008-09-08 at 16:27 -0500, Paul B Schroeder wrote:
> The powers that be require us to use Perforce. Thus, the patch. A few
> things to note about it:
>
> * Our P4 server is running on a non-standard port and thus needs to be
> specified. scm_tuple is now created via rsplit instead. Allows SCM
> port to be specified. i.e. This now works:
> allowed_scms=perforcetx.bluecoat.com:1999:/koji/sandbox/paul.schroeder/skeletor
> * P4 *requires* a stinkin' password. So I added password support to
> _parse_url. The "user" part of the SCM url can now be "user:password".
> Still works minus the ":password" too. Let me know if there's a
> better/preferred way to support passwords.
> * P4 can only checkout into the static client root specified in the P4
> client spec (PITA). Sooo.. After the module_checkout_cmd is executed,
> if sourcedir does not exist, we do a 'os.rename' of the checked out code
> into the proper task sourcedir.
> * A P4 SCM URL looks like:
> p4://user:pass@p4port/p4client/p4depot?path/to/module#changelist
> i.e. p4port==host And p4client is your P4 client spec that you've
> created.
>
> Let me know if there are any Qs, comments, or any changes I should
> make..

The patch is linewrapped; any chance you could repost and make sure to
use the "preformat" setting or something in your mail client?

Dan

> Cheers...Paul...
>
>
> diff --git a/builder/kojid b/builder/kojid
> index 1a7afbf..92f8143 100755
> --- a/builder/kojid
> +++ b/builder/kojid
> @@ -1830,7 +1830,8 @@ class BuildSRPMFromSCMTask(BaseTaskHandler):
> use_common = True
>
> for allowed_scm in options.allowed_scms.split():
> - scm_tuple = allowed_scm.split(':')
> + # Use rsplit with 1 max split to allow for port numbers w/
> scm host
> + scm_tuple = allowed_scm.rsplit(':', 1)
> if len(scm_tuple) in (2, 3):
> if fnmatch(scm.host, scm_tuple[0]) and
> fnmatch(scm.repository, scm_tuple[1]):
> # SCM host:repository is in the allowed list
> @@ -2325,7 +2326,8 @@ class SCM(object):
> 'GIT': ('git://', 'git+http://', 'git+https://', 'git
> +rsync://'),
> 'GIT+SSH': ('git+ssh://',),
> 'SVN': ('svn://', 'svn+http://', 'svn+https://'),
> - 'SVN+SSH': ('svn+ssh://',) }
> + 'SVN+SSH': ('svn+ssh://',),
> + 'P4': ('p4://',) }
>
> def is_scm_url(url):
> """
> @@ -2362,10 +2364,11 @@ class SCM(object):
> raise koji.GenericError, 'Invalid SCM URL: %s' % url
>
> self.url = url
> - scheme, user, host, path, query, fragment = self._parse_url()
> + scheme, user, password, host, path, query, fragment =
> self._parse_url()
>
> self.scheme = scheme
> self.user = user
> + self.password = password
> self.host = host
> self.repository = path
> self.module = query
> @@ -2384,9 +2387,9 @@ class SCM(object):
> Parse the SCM url into usable components.
> Return the following tuple:
>
> - (scheme, user, host, path, query, fragment)
> + (scheme, user, password, host, path, query, fragment)
>
> - user may be None, everything else will have a value
> + user and password may be None, everything else will have a
> value
> """
> # get the url's scheme
> scheme = self.url.split('://')[0] + '://'
> @@ -2396,14 +2399,21 @@ class SCM(object):
> dummyscheme, netloc, path, params, query, fragment =
> urlparse.urlparse(dummyurl)
>
> user = None
> + password = None
> userhost = netloc.split('@')
> if len(userhost) == 2:
> - user = userhost[0]
> + userpass = userhost[0].split(':')
> + user = userpass[0]
> + if len(userpass) == 2:
> + password = userpass[1]
> + elif len(userpass) > 2:
> + raise koji.GenericError, 'Invalid username:password
> specified: %s' % netloc
> if not user:
> # Don't return an empty string
> user = None
> - elif ':' in user:
> - raise koji.GenericError, 'username:password format not
> supported: %s' % user
> + if not password:
> + # Don't return an empty string
> + password = None
> netloc = userhost[1]
> elif len(userhost) > 2:
> raise koji.GenericError, 'Invalid username@hostname
> specified: %s' % netloc
> @@ -2419,7 +2429,7 @@ class SCM(object):
> raise koji.GenericError, 'Unable to parse SCM URL: %s' %
> self.url
>
> # return parsed values
> - return (scheme, user, netloc, path, query, fragment)
> + return (scheme, user, password, netloc, path, query, fragment)
>
> def checkout(self, scmdir, uploadpath, logfile, use_common=False):
> """
> @@ -2514,6 +2524,39 @@ class SCM(object):
> module_checkout_cmd = ['svn', 'checkout', '-r',
> self.revision, '%s/%s' % (svnserver, self.module), self.module]
> common_checkout_cmd = ['svn', 'checkout', '%s/common' %
> svnserver]
>
> + elif self.scmtype == 'P4':
> + if not self.user:
> + raise koji.BuildError, 'No user specified for
> repository access scheme: %s' % self.scheme
> + # P4 URL:
> + #
> p4://user:pass@p4port/p4client/p4depot?path/to/module#changelist
> +
> + # Perforce uses a "client workspace"
> + # Before first slash is the p4 client, after is the p4
> depot
> + (p4client, p4depot) = self.repository[1:].split('/', 1)
> +
> + p4client_root = None
> + p4info = os.popen('p4 -p ' + self.host + ' -c ' + p4client
> + ' info')
> + for line in p4info:
> + if line.startswith('Client root:'):
> + p4client_root = line.split()[-1]
> + break
> + p4info.close()
> + if not p4client_root:
> + raise koji.BuildError, 'Could not find perforce client
> root'
> +
> + # Perforce cannot checkout to a random dir (in this case
> scmdir).
> + # It will only checkout to the p4client_root, so we
> override scmdir
> + # with the p4client_root
> + scmdir = '%s/%s' % (p4client_root, p4depot)
> +
> + # When HEAD is given as the revision, we assume we want the
> latest
> + p4revision = '
> + if self.revision.upper() != 'HEAD':
> + p4revision = '@%s' % self.revision
> +
> + module_checkout_cmd = ['p4', '-p', self.host, '-u',
> self.user, '-P', self.password, '-c', p4client, 'sync', '-f', '//%s/%
> s/...%s' % (p4depot, self.module, p4revision)]
> + common_checkout_cmd = ['p4', '-p', self.host, '-u',
> self.user, '-P', self.password, '-c', p4client, 'sync', '-f', '//%
> s/common/...' % p4depot]
> +
> else:
> raise koji.BuildError, 'Unknown SCM type: %s' %
> self.scmtype
>
> @@ -2521,6 +2564,10 @@ class SCM(object):
> if log_output(module_checkout_cmd[0], module_checkout_cmd,
> logfile, uploadpath, cwd=scmdir, logerror=1, env=env):
> raise koji.BuildError, 'Error running %s checkout command
> "%s", see %s for details' %
> (self.scmtype, ' '.join(module_checkout_cmd),
> os.path.basename(logfile))
> + # Perforce checks out to the client root listed for the
> perforce client.
> + # Need to move checked out source into sourcedir.
> + if not os.path.exists(sourcedir):
> + os.rename('%s/%s' % (scmdir, self.module), sourcedir)
>
> if update_checkout_cmd:
> # Currently only required for GIT checkouts
>
>
> --
> Fedora-buildsys-list mailing list
> Fedora-buildsys-list@redhat.com
> https://www.redhat.com/mailman/listinfo/fedora-buildsys-list

--
Fedora-buildsys-list mailing list
Fedora-buildsys-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-buildsys-list

Paul B Schroeder 09-08-2008 10:32 PM

Perforce support for koji
 
On Mon, 2008-09-08 at 17:33 -0400, Dan Williams wrote:
> The patch is linewrapped; any chance you could repost and make sure to
> use the "preformat" setting or something in your mail client?
Sure can.. Sorry about that. Also, quickly, this version fixes the previous patch
in that the scm_tuple can handle when you specify use_common with an allowed_csm:
allowed_scms=perforcetx.bluecoat.com:1999:/koji/sandbox/paul.schroeder/skeletor
allowed_scms=perforcetx.bluecoat.com:1999:/koji/sandbox/paul.schroeder/skeletor:false
allowed_scms=perforcetx.bluecoat.com:1999:/koji/sandbox/paul.schroeder/skeletor:true

Cheers...Paul...


diff --git a/builder/kojid b/builder/kojid
index 1a7afbf..8f510f6 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -1830,14 +1830,19 @@ class BuildSRPMFromSCMTask(BaseTaskHandler):
use_common = True

for allowed_scm in options.allowed_scms.split():
- scm_tuple = allowed_scm.split(':')
- if len(scm_tuple) in (2, 3):
+ scm_tuple = allowed_scm.rsplit(':', 1)
+ # check if we specify a value for use_common
+ if scm_tuple[1].lower() in ('false', 'no', '0'):
+ use_common = False
+ scm_tuple = scm_tuple[0].rsplit(':', 1)
+ elif scm_tuple[1].lower() in ('true', 'yes', '1'):
+ use_common = True
+ scm_tuple = scm_tuple[0].rsplit(':', 1)
+ else:
+ use_common = True
+ if len(scm_tuple) == 2:
if fnmatch(scm.host, scm_tuple[0]) and fnmatch(scm.repository, scm_tuple[1]):
# SCM host:repository is in the allowed list
- # check if we specify a value for use_common
- if len(scm_tuple) == 3:
- if scm_tuple[2].lower() in ('false', 'no', '0'):
- use_common = False
break
else:
self.logger.warn('Ignoring incorrectly formatted SCM host:repository: %s' % allowed_scm)
@@ -2325,7 +2330,8 @@ class SCM(object):
'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'),
'GIT+SSH': ('git+ssh://',),
'SVN': ('svn://', 'svn+http://', 'svn+https://'),
- 'SVN+SSH': ('svn+ssh://',) }
+ 'SVN+SSH': ('svn+ssh://',),
+ 'P4': ('p4://',) }

def is_scm_url(url):
"""
@@ -2362,10 +2368,11 @@ class SCM(object):
raise koji.GenericError, 'Invalid SCM URL: %s' % url

self.url = url
- scheme, user, host, path, query, fragment = self._parse_url()
+ scheme, user, password, host, path, query, fragment = self._parse_url()

self.scheme = scheme
self.user = user
+ self.password = password
self.host = host
self.repository = path
self.module = query
@@ -2384,9 +2391,9 @@ class SCM(object):
Parse the SCM url into usable components.
Return the following tuple:

- (scheme, user, host, path, query, fragment)
+ (scheme, user, password, host, path, query, fragment)

- user may be None, everything else will have a value
+ user and password may be None, everything else will have a value
"""
# get the url's scheme
scheme = self.url.split('://')[0] + '://'
@@ -2396,14 +2403,21 @@ class SCM(object):
dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl)

user = None
+ password = None
userhost = netloc.split('@')
if len(userhost) == 2:
- user = userhost[0]
+ userpass = userhost[0].split(':')
+ user = userpass[0]
+ if len(userpass) == 2:
+ password = userpass[1]
+ elif len(userpass) > 2:
+ raise koji.GenericError, 'Invalid username:password specified: %s' % netloc
if not user:
# Don't return an empty string
user = None
- elif ':' in user:
- raise koji.GenericError, 'username:password format not supported: %s' % user
+ if not password:
+ # Don't return an empty string
+ password = None
netloc = userhost[1]
elif len(userhost) > 2:
raise koji.GenericError, 'Invalid username@hostname specified: %s' % netloc
@@ -2419,7 +2433,7 @@ class SCM(object):
raise koji.GenericError, 'Unable to parse SCM URL: %s' % self.url

# return parsed values
- return (scheme, user, netloc, path, query, fragment)
+ return (scheme, user, password, netloc, path, query, fragment)

def checkout(self, scmdir, uploadpath, logfile, use_common=False):
"""
@@ -2514,6 +2528,39 @@ class SCM(object):
module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
common_checkout_cmd = ['svn', 'checkout', '%s/common' % svnserver]

+ elif self.scmtype == 'P4':
+ if not self.user:
+ raise koji.BuildError, 'No user specified for repository access scheme: %s' % self.scheme
+ # P4 URL:
+ # p4://user:pass@p4port/p4client/p4depot?path/to/module#changelist
+
+ # Perforce uses a "client workspace"
+ # Before first slash is the p4 client, after is the p4 depot
+ (p4client, p4depot) = self.repository[1:].split('/', 1)
+
+ p4client_root = None
+ p4info = os.popen('p4 -p ' + self.host + ' -c ' + p4client + ' info')
+ for line in p4info:
+ if line.startswith('Client root:'):
+ p4client_root = line.split()[-1]
+ break
+ p4info.close()
+ if not p4client_root:
+ raise koji.BuildError, 'Could not find perforce client root'
+
+ # Perforce cannot checkout to a random dir (in this case scmdir).
+ # It will only checkout to the p4client_root, so we override scmdir
+ # with the p4client_root
+ scmdir = '%s/%s' % (p4client_root, p4depot)
+
+ # When HEAD is given as the revision, we assume we want the latest
+ p4revision = '
+ if self.revision.upper() != 'HEAD':
+ p4revision = '@%s' % self.revision
+
+ module_checkout_cmd = ['p4', '-p', self.host, '-u', self.user, '-P', self.password, '-c', p4client, 'sync', '-f', '//%s/%s/...%s' % (p4depot, self.module, p4revision)]
+ common_checkout_cmd = ['p4', '-p', self.host, '-u', self.user, '-P', self.password, '-c', p4client, 'sync', '-f', '//%s/common/...' % p4depot]
+
else:
raise koji.BuildError, 'Unknown SCM type: %s' % self.scmtype

@@ -2521,6 +2568,10 @@ class SCM(object):
if log_output(module_checkout_cmd[0], module_checkout_cmd, logfile, uploadpath, cwd=scmdir, logerror=1, env=env):
raise koji.BuildError, 'Error running %s checkout command "%s", see %s for details' %
(self.scmtype, ' '.join(module_checkout_cmd), os.path.basename(logfile))
+ # Perforce checks out to the client root listed for the perforce client.
+ # Need to move checked out source into sourcedir.
+ if not os.path.exists(sourcedir):
+ os.rename('%s/%s' % (scmdir, self.module), sourcedir)

if update_checkout_cmd:
# Currently only required for GIT checkouts


--
Fedora-buildsys-list mailing list
Fedora-buildsys-list@redhat.com
https://www.redhat.com/mailman/listinfo/fedora-buildsys-list


All times are GMT. The time now is 09:27 PM.

VBulletin, Copyright ©2000 - 2014, Jelsoft Enterprises Ltd.
Content Relevant URLs by vBSEO ©2007, Crawlability, Inc.