rust/hg-cpython/src/copy_tracing.rs
author Arseniy Alekseyev <aalekseyev@janestreet.com>
Wed, 04 Jan 2023 19:13:41 +0000
changeset 49886 3aa8e569478a
parent 47953 8f031a274cd6
child 49933 be3b545c5cff
permissions -rw-r--r--
merge: don't pay for pathconflicts if there are none
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     1
use cpython::ObjectProtocol;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     2
use cpython::PyBytes;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     3
use cpython::PyDict;
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
     4
use cpython::PyDrop;
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     5
use cpython::PyList;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     6
use cpython::PyModule;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     7
use cpython::PyObject;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     8
use cpython::PyResult;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     9
use cpython::PyTuple;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    10
use cpython::Python;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    11
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    12
use hg::copy_tracing::ChangedFiles;
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    13
use hg::copy_tracing::CombineChangesetCopies;
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    14
use hg::Revision;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    15
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
    16
use crate::pybytes_deref::PyBytesDeref;
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    17
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    18
/// Combines copies information contained into revision `revs` to build a copy
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    19
/// map.
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    20
///
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    21
/// See mercurial/copies.py for details
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    22
pub fn combine_changeset_copies_wrapper(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    23
    py: Python,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    24
    revs: PyList,
46149
294d5aca4ff5 copies: iterate over children directly (instead of parents)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46057
diff changeset
    25
    children_count: PyDict,
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    26
    target_rev: Revision,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    27
    rev_info: PyObject,
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    28
    multi_thread: bool,
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    29
) -> PyResult<PyDict> {
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    30
    let children_count = children_count
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    31
        .items(py)
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    32
        .iter()
46149
294d5aca4ff5 copies: iterate over children directly (instead of parents)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46057
diff changeset
    33
        .map(|(k, v)| Ok((k.extract(py)?, v.extract(py)?)))
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    34
        .collect::<PyResult<_>>()?;
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    35
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    36
    /// (Revision number, parent 1, parent 2, copy data for this revision)
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    37
    type RevInfo<Bytes> = (Revision, Revision, Revision, Option<Bytes>);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    38
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    39
    let revs_info =
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    40
        revs.iter(py).map(|rev_py| -> PyResult<RevInfo<PyBytes>> {
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    41
            let rev = rev_py.extract(py)?;
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    42
            let tuple: PyTuple =
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    43
                rev_info.call(py, (rev_py,), None)?.cast_into(py)?;
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    44
            let p1 = tuple.get_item(py, 0).extract(py)?;
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    45
            let p2 = tuple.get_item(py, 1).extract(py)?;
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    46
            let opt_bytes = tuple.get_item(py, 2).extract(py)?;
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    47
            Ok((rev, p1, p2, opt_bytes))
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    48
        });
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    49
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    50
    let path_copies;
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    51
    if !multi_thread {
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    52
        let mut combine_changeset_copies =
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    53
            CombineChangesetCopies::new(children_count);
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    54
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    55
        for rev_info in revs_info {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    56
            let (rev, p1, p2, opt_bytes) = rev_info?;
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    57
            let files = match &opt_bytes {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    58
                Some(bytes) => ChangedFiles::new(bytes.data(py)),
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    59
                // Python None was extracted to Option::None,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    60
                // meaning there was no copy data.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    61
                None => ChangedFiles::new_empty(),
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    62
            };
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    63
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    64
            combine_changeset_copies.add_revision(rev, p1, p2, files)
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    65
        }
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    66
        path_copies = combine_changeset_copies.finish(target_rev)
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    67
    } else {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    68
        // Use a bounded channel to provide back-pressure:
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    69
        // if the child thread is slower to process revisions than this thread
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    70
        // is to gather data for them, an unbounded channel would keep
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    71
        // growing and eat memory.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    72
        //
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    73
        // TODO: tweak the bound?
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    74
        let (rev_info_sender, rev_info_receiver) =
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
    75
            crossbeam_channel::bounded::<RevInfo<PyBytesDeref>>(1000);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    76
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    77
        // This channel (going the other way around) however is unbounded.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    78
        // If they were both bounded, there might potentially be deadlocks
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    79
        // where both channels are full and both threads are waiting on each
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    80
        // other.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    81
        let (pybytes_sender, pybytes_receiver) =
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    82
            crossbeam_channel::unbounded();
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    83
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    84
        // Start a thread that does CPU-heavy processing in parallel with the
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    85
        // loop below.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    86
        //
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    87
        // If the parent thread panics, `rev_info_sender` will be dropped and
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    88
        // “disconnected”. `rev_info_receiver` will be notified of this and
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    89
        // exit its own loop.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    90
        let thread = std::thread::spawn(move || {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    91
            let mut combine_changeset_copies =
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    92
                CombineChangesetCopies::new(children_count);
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    93
            for (rev, p1, p2, opt_bytes) in rev_info_receiver {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    94
                let files = match &opt_bytes {
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
    95
                    Some(raw) => ChangedFiles::new(raw.as_ref()),
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    96
                    // Python None was extracted to Option::None,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    97
                    // meaning there was no copy data.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    98
                    None => ChangedFiles::new_empty(),
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    99
                };
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   100
                combine_changeset_copies.add_revision(rev, p1, p2, files);
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
   101
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   102
                // Send `PyBytes` back to the parent thread so the parent
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   103
                // thread can drop it. Otherwise the GIL would be implicitly
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   104
                // acquired here through `impl Drop for PyBytes`.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   105
                if let Some(bytes) = opt_bytes {
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   106
                    if let Err(_) = pybytes_sender.send(bytes.unwrap()) {
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   107
                        // The channel is disconnected, meaning the parent
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   108
                        // thread panicked or returned
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   109
                        // early through
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   110
                        // `?` to propagate a Python exception.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   111
                        break;
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   112
                    }
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   113
                }
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   114
            }
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   115
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   116
            combine_changeset_copies.finish(target_rev)
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   117
        });
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
   118
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   119
        for rev_info in revs_info {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   120
            let (rev, p1, p2, opt_bytes) = rev_info?;
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
   121
            let opt_bytes = opt_bytes.map(|b| PyBytesDeref::new(py, b));
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   122
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   123
            // We’d prefer to avoid the child thread calling into Python code,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   124
            // but this avoids a potential deadlock on the GIL if it does:
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   125
            py.allow_threads(|| {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   126
                rev_info_sender.send((rev, p1, p2, opt_bytes)).expect(
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   127
                    "combine_changeset_copies: channel is disconnected",
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   128
                );
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   129
            });
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   130
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   131
            // Drop anything in the channel, without blocking
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   132
            for pybytes in pybytes_receiver.try_iter() {
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   133
                pybytes.release_ref(py)
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   134
            }
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   135
        }
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   136
        // We’d prefer to avoid the child thread calling into Python code,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   137
        // but this avoids a potential deadlock on the GIL if it does:
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   138
        path_copies = py.allow_threads(|| {
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   139
            // Disconnect the channel to signal the child thread to stop:
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   140
            // the `for … in rev_info_receiver` loop will end.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   141
            drop(rev_info_sender);
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   142
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   143
            // Wait for the child thread to stop, and propagate any panic.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   144
            thread.join().unwrap_or_else(|panic_payload| {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   145
                std::panic::resume_unwind(panic_payload)
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   146
            })
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   147
        });
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   148
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   149
        // Drop anything left in the channel
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   150
        for pybytes in pybytes_receiver.iter() {
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   151
            pybytes.release_ref(py)
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   152
        }
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   153
    };
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   154
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   155
    let out = PyDict::new(py);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
   156
    for (dest, source) in path_copies.into_iter() {
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   157
        out.set_item(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   158
            py,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   159
            PyBytes::new(py, &dest.into_vec()),
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   160
            PyBytes::new(py, &source.into_vec()),
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   161
        )?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   162
    }
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   163
    Ok(out)
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   164
}
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   165
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   166
/// Create the module, with `__package__` given from parent
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   167
pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   168
    let dotted_name = &format!("{}.copy_tracing", package);
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   169
    let m = PyModule::new(py, dotted_name)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   170
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   171
    m.add(py, "__package__", package)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   172
    m.add(py, "__doc__", "Copy tracing - Rust implementation")?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   173
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   174
    m.add(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   175
        py,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   176
        "combine_changeset_copies",
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   177
        py_fn!(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   178
            py,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   179
            combine_changeset_copies_wrapper(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   180
                revs: PyList,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   181
                children: PyDict,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   182
                target_rev: Revision,
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   183
                rev_info: PyObject,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   184
                multi_thread: bool
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   185
            )
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   186
        ),
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   187
    )?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   188
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   189
    let sys = PyModule::import(py, "sys")?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   190
    let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   191
    sys_modules.set_item(py, dotted_name, &m)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   192
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   193
    Ok(m)
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   194
}