# HG changeset patch # User Bryan O'Sullivan # Date 1338407733 25200 # Node ID 92e1c64ba0d481682181e809b68b389d0dac31d6 # Parent 8f36806b8f6a6e235dba609298a4855967afe517 parsers: add a C function to pack the dirstate This is about 9 times faster than the Python dirstate packing code. The relatively small speedup is due to the poor locality and memory access patterns caused by traversing dicts and other boxed Python values. diff -r 8f36806b8f6a -r 92e1c64ba0d4 mercurial/dirstate.py --- a/mercurial/dirstate.py Fri Jun 08 05:31:28 2012 +0300 +++ b/mercurial/dirstate.py Wed May 30 12:55:33 2012 -0700 @@ -498,12 +498,24 @@ return st = self._opener("dirstate", "w", atomictemp=True) + def finish(s): + st.write(s) + st.close() + self._lastnormaltime = 0 + self._dirty = self._dirtypl = False + # use the modification time of the newly created temporary file as the # filesystem's notion of 'now' - now = int(util.fstat(st).st_mtime) + now = util.fstat(st).st_mtime + copymap = self._copymap + try: + finish(parsers.pack_dirstate(self._map, copymap, self._pl, now)) + return + except AttributeError: + pass + now = int(now) cs = cStringIO.StringIO() - copymap = self._copymap pack = struct.pack write = cs.write write("".join(self._pl)) @@ -526,10 +538,7 @@ e = pack(_format, e[0], e[1], e[2], e[3], len(f)) write(e) write(f) - st.write(cs.getvalue()) - st.close() - self._lastnormaltime = 0 - self._dirty = self._dirtypl = False + finish(cs.getvalue()) def _dirignore(self, f): if f == '.': diff -r 8f36806b8f6a -r 92e1c64ba0d4 mercurial/parsers.c --- a/mercurial/parsers.c Fri Jun 08 05:31:28 2012 +0300 +++ b/mercurial/parsers.c Wed May 30 12:55:33 2012 -0700 @@ -214,6 +214,154 @@ return ret; } +static inline int getintat(PyObject *tuple, int off, uint32_t *v) +{ + PyObject *o = PyTuple_GET_ITEM(tuple, off); + long val; + + if (PyInt_Check(o)) + val = PyInt_AS_LONG(o); + else if (PyLong_Check(o)) { + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) + return -1; + } else { + PyErr_SetString(PyExc_TypeError, "expected an int or long"); + return -1; + } + if (LONG_MAX > INT_MAX && (val > INT_MAX || val < INT_MIN)) { + PyErr_SetString(PyExc_OverflowError, + "Python value to large to convert to uint32_t"); + return -1; + } + *v = (uint32_t)val; + return 0; +} + +static PyObject *dirstate_unset; + +/* + * Efficiently pack a dirstate object into its on-disk format. + */ +static PyObject *pack_dirstate(PyObject *self, PyObject *args) +{ + PyObject *packobj = NULL; + PyObject *map, *copymap, *pl; + Py_ssize_t nbytes, pos, l; + PyObject *k, *v, *pn; + char *p, *s; + double now; + + if (!PyArg_ParseTuple(args, "O!O!Od:pack_dirstate", + &PyDict_Type, &map, &PyDict_Type, ©map, + &pl, &now)) + return NULL; + + if (!PySequence_Check(pl) || PySequence_Size(pl) != 2) { + PyErr_SetString(PyExc_TypeError, "expected 2-element sequence"); + return NULL; + } + + /* Figure out how much we need to allocate. */ + for (nbytes = 40, pos = 0; PyDict_Next(map, &pos, &k, &v);) { + PyObject *c; + if (!PyString_Check(k)) { + PyErr_SetString(PyExc_TypeError, "expected string key"); + goto bail; + } + nbytes += PyString_GET_SIZE(k) + 17; + c = PyDict_GetItem(copymap, k); + if (c) { + if (!PyString_Check(c)) { + PyErr_SetString(PyExc_TypeError, + "expected string key"); + goto bail; + } + nbytes += PyString_GET_SIZE(c) + 1; + } + } + + packobj = PyString_FromStringAndSize(NULL, nbytes); + if (packobj == NULL) + goto bail; + + p = PyString_AS_STRING(packobj); + + pn = PySequence_ITEM(pl, 0); + if (PyString_AsStringAndSize(pn, &s, &l) == -1 || l != 20) { + PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash"); + goto bail; + } + memcpy(p, s, l); + p += 20; + pn = PySequence_ITEM(pl, 1); + if (PyString_AsStringAndSize(pn, &s, &l) == -1 || l != 20) { + PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash"); + goto bail; + } + memcpy(p, s, l); + p += 20; + + for (pos = 0; PyDict_Next(map, &pos, &k, &v); ) { + uint32_t mode, size, mtime; + Py_ssize_t len, l; + PyObject *o; + char *s, *t; + int err; + + if (!PyTuple_Check(v) || PyTuple_GET_SIZE(v) != 4) { + PyErr_SetString(PyExc_TypeError, "expected a 4-tuple"); + goto bail; + } + o = PyTuple_GET_ITEM(v, 0); + if (PyString_AsStringAndSize(o, &s, &l) == -1 || l != 1) { + PyErr_SetString(PyExc_TypeError, "expected one byte"); + goto bail; + } + *p++ = *s; + err = getintat(v, 1, &mode); + err |= getintat(v, 2, &size); + err |= getintat(v, 3, &mtime); + if (err) + goto bail; + if (*s == 'n' && mtime == (uint32_t)now) { + /* See dirstate.py:write for why we do this. */ + if (PyDict_SetItem(map, k, dirstate_unset) == -1) + goto bail; + mode = 0, size = -1, mtime = -1; + } + putbe32(mode, p); + putbe32(size, p + 4); + putbe32(mtime, p + 8); + t = p + 12; + p += 16; + len = PyString_GET_SIZE(k); + memcpy(p, PyString_AS_STRING(k), len); + p += len; + o = PyDict_GetItem(copymap, k); + if (o) { + *p++ = '\0'; + l = PyString_GET_SIZE(o); + memcpy(p, PyString_AS_STRING(o), l); + p += l; + len += l + 1; + } + putbe32((uint32_t)len, t); + } + + pos = p - PyString_AS_STRING(packobj); + if (pos != nbytes) { + PyErr_Format(PyExc_SystemError, "bad dirstate size: %ld != %ld", + (long)pos, (long)nbytes); + goto bail; + } + + return packobj; +bail: + Py_XDECREF(packobj); + return NULL; +} + /* * A base-16 trie for fast node->rev mapping. * @@ -1356,6 +1504,7 @@ static char parsers_doc[] = "Efficient content parsing."; static PyMethodDef methods[] = { + {"pack_dirstate", pack_dirstate, METH_VARARGS, "pack a dirstate\n"}, {"parse_manifest", parse_manifest, METH_VARARGS, "parse a manifest\n"}, {"parse_dirstate", parse_dirstate, METH_VARARGS, "parse a dirstate\n"}, {"parse_index2", parse_index2, METH_VARARGS, "parse a revlog index\n"}, @@ -1375,6 +1524,8 @@ -1, -1, -1, -1, nullid, 20); if (nullentry) PyObject_GC_UnTrack(nullentry); + + dirstate_unset = Py_BuildValue("ciii", 'n', 0, -1, -1); } #ifdef IS_PY3K