fabric/pyvenvmanager.cpp

fabric/pyvenvmanager.cpp

fabric/pyvenvmanager.cpp

Namespaces

Name
Syntalos

Source code

/*
 * Copyright (C) 2020-2026 Matthias Klumpp <matthias@tenstral.net>
 *
 * Licensed under the GNU Lesser General Public License Version 3
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the license, or
 * (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this software.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "pyvenvmanager.h"

#include <QDebug>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QStandardPaths>
#include <QTextStream>

#include "executils.h"
#include "globalconfig.h"
#include "utils/misc.h"

namespace Syntalos
{

Q_LOGGING_CATEGORY(logVEnv, "pyvenv")

QString pythonVEnvDirForName(const QString &venvName)
{
    GlobalConfig gconf;
    return QStringLiteral("%1/%2").arg(gconf.virtualEnvDir(), venvName);
}

static bool pythonVirtualEnvExists(const QString &venvName)
{
    return QFile::exists(QStringLiteral("%1/bin/python").arg(pythonVEnvDirForName(venvName)));
}

static QString requirementsHashMetadataPath(const QString &venvDir)
{
    return QStringLiteral("%1/.requirements.b3sum").arg(venvDir);
}

static bool writeRequirementsHashMetadata(const QString &venvDir, const QByteArray &requirementsHash)
{
    const auto metadataPath = requirementsHashMetadataPath(venvDir);
    if (requirementsHash.isEmpty()) {
        QFile::remove(metadataPath);
        return true;
    }

    QFile metadataFile(metadataPath);
    if (!metadataFile.open(QIODevice::WriteOnly | QIODevice::Text)) {
        qCWarning(logVEnv).noquote() << "Unable to write venv requirements metadata:" << metadataPath;
        return false;
    }

    QTextStream out(&metadataFile);
    out << QString::fromLatin1(requirementsHash.toHex()) << "\n";
    return true;
}

static bool readRequirementsHashMetadata(const QString &venvDir, QString &requirementsHashOut)
{
    requirementsHashOut = QString();

    QFile metadataFile(requirementsHashMetadataPath(venvDir));
    if (!metadataFile.exists())
        return false;

    if (!metadataFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
        qCWarning(logVEnv).noquote() << "Unable to read venv requirements metadata for:" << venvDir;
        return false;
    }

    const auto rawHash = QString::fromUtf8(metadataFile.readAll()).trimmed();
    if (rawHash.isEmpty())
        return false;

    requirementsHashOut = rawHash;
    return true;
}

static QString interpreterPathFromCfg(const QString &venvDir)
{
    QFile cfgFile(QStringLiteral("%1/pyvenv.cfg").arg(venvDir));
    if (!cfgFile.open(QIODevice::ReadOnly | QIODevice::Text))
        return QString();

    QString executablePath;
    QString home;

    QTextStream in(&cfgFile);
    while (!in.atEnd()) {
        const auto line = in.readLine();
        const auto eqPos = line.indexOf('=');
        if (eqPos <= 0)
            continue;

        const auto key = line.left(eqPos).trimmed();
        const auto value = line.mid(eqPos + 1).trimmed();
        if (key == QStringLiteral("executable"))
            executablePath = value;
        else if (key == QStringLiteral("home"))
            home = value;
    }

    if (!executablePath.isEmpty())
        return executablePath;

    if (!home.isEmpty()) {
        const auto py3Path = QStringLiteral("%1/python3").arg(home);
        if (QFileInfo(py3Path).isAbsolute())
            return py3Path;

        const auto pyPath = QStringLiteral("%1/python").arg(home);
        if (QFileInfo(pyPath).isAbsolute())
            return pyPath;
    }

    return QString();
}

PyVirtualEnvStatus pythonVirtualEnvStatus(const QString &venvName, const QString &requirementsFile)
{
    if (!pythonVirtualEnvExists(venvName))
        return PyVirtualEnvStatus::MISSING;

    const auto venvDir = pythonVEnvDirForName(venvName);
    const auto interpreterPath = interpreterPathFromCfg(venvDir);
    if (!interpreterPath.isEmpty() && QFileInfo(interpreterPath).isAbsolute() && !QFileInfo::exists(interpreterPath)) {
        qCWarning(logVEnv).noquote() << "Base Python interpreter for virtual environment" << venvName
                                     << "is missing:" << interpreterPath;
        return PyVirtualEnvStatus::INTERPRETER_MISSING;
    }

    if (requirementsFile.isEmpty())
        return PyVirtualEnvStatus::VALID;

    const auto hashResult = blake3HashForFile(requirementsFile);
    QString currentHashStr;
    if (hashResult.has_value())
        currentHashStr = QString::fromLatin1(hashResult->toHex());
    else
        qCWarning(logVEnv).noquote() << "Unable to compute BLAKE3 hash for requirements file:" << requirementsFile
                                     << "- treating as changed requirements.";

    QString storedReqHash;
    if (!readRequirementsHashMetadata(venvDir, storedReqHash)) {
        qCWarning(logVEnv).noquote() << "Unable to read stored requirements hash metadata for virtual environment"
                                     << venvName << "- treating as changed requirements.";
        return PyVirtualEnvStatus::REQUIREMENTS_CHANGED;
    }

    if (storedReqHash != currentHashStr)
        return PyVirtualEnvStatus::REQUIREMENTS_CHANGED;

    return PyVirtualEnvStatus::VALID;
}

static void injectSystemPyModule(const QString &venvDir, const QString &pyModName)
{
    QDir dir(QStringLiteral("%1/lib/").arg(venvDir));
    QStringList vpDirs = dir.entryList(QStringList() << QStringLiteral("python*"));

    QStringList systemPyModPaths = QStringList()
                                   << QStringLiteral("/usr/lib/python3/dist-packages/%1/").arg(pyModName)
                                   << QStringLiteral("/usr/local/lib/python3/dist-packages/%1/").arg(pyModName)
                                   << QStringLiteral("/app/lib/python/site-packages/%1/").arg(pyModName);

    for (const auto &systemPyQtPath : systemPyModPaths) {
        QDir sysPyQtDir(systemPyQtPath);
        if (!sysPyQtDir.exists())
            continue;

        for (const auto &pyDir : vpDirs) {
            qDebug().noquote() << "Adding system Python module to venv:" << systemPyQtPath;
            QFile::link(systemPyQtPath, QStringLiteral("%1/lib/%2/site-packages/%3").arg(venvDir, pyDir, pyModName));
        }
    }
}

static bool injectSystemPyQtBindings(const QString &venvDir)
{
    // the PyQt6/PySide modules must be for the same version as Syntalos was compiled for,
    // as the pyworker binary is linked against Qt as well (and trying to load a different
    // version will fail)
    // therefore, we add this hack and inject just the system Qt Python bindings into the
    // virtual environment.
    injectSystemPyModule(venvDir, QStringLiteral("PyQt6"));
    return true;
}

auto createPythonVirtualEnv(const QString &venvName, const QString &requirementsFile, bool recreate)
    -> std::expected<QString, QString>
{
    auto rtdDir = QStandardPaths::writableLocation(QStandardPaths::RuntimeLocation);
    if (rtdDir.isEmpty())
        rtdDir = "/tmp";
    const auto venvDir = pythonVEnvDirForName(venvName);

    if (QDir(venvDir).exists()) {
        if (recreate) {
            qCDebug(logVEnv).noquote() << "Removing existing virtualenv before recreation:" << venvDir;
            if (!QDir(venvDir).removeRecursively())
                return std::unexpected(QStringLiteral("Unable to remove existing virtualenv: %1").arg(venvDir));
        } else {
            // nothing to do, the venv already exists
            return venvDir;
        }
    }

    QDir().mkpath(venvDir);

    const auto tmpCommandFile = QStringLiteral("%1/sy-venv-%2.sh").arg(rtdDir, createRandomString(6));
    QFile shFile(tmpCommandFile);
    if (!shFile.open(QIODevice::WriteOnly | QIODevice::Text)) {
        qCWarning(logVEnv).noquote() << "Unable to open temporary file" << tmpCommandFile << "for writing.";
        return std::unexpected(
            QStringLiteral("Unable to open temporary file for virtualenv creation: %1").arg(shFile.errorString()));
    }

    const auto tmpRequirementsFname = QStringLiteral("%1/%2-requirements_%3.txt")
                                          .arg(rtdDir, venvName, createRandomString(4));
    QFile tmpReqFile(tmpRequirementsFname);
    if (!tmpReqFile.open(QIODevice::WriteOnly | QIODevice::Text)) {
        qCWarning(logVEnv).noquote() << "Unable to open temporary file" << tmpRequirementsFname << "for writing.";
        return std::unexpected(
            QStringLiteral("Unable to open temporary file for virtualenv creation: %1").arg(tmpReqFile.errorString()));
    }

    if (requirementsFile.isEmpty()) {
        // we have no requirements file and want to create a blank venv, so we just create an empty dummy file
        QTextStream rqfOut(&tmpReqFile);
        rqfOut << "\n";
        tmpReqFile.close();
        qCDebug(logVEnv) << "Creating empty virtualenv" << venvName;
    } else {
        QFile reqFile(requirementsFile);
        if (!reqFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
            qCWarning(logVEnv).noquote() << "Unable to open file" << requirementsFile << "for reading.";
            return std::unexpected(QStringLiteral("Unable to open requirements file \"%1\": %2")
                                       .arg(requirementsFile, reqFile.errorString()));
        }

        QTextStream rqfIn(&reqFile);
        QTextStream rqfOut(&tmpReqFile);
        while (!rqfIn.atEnd()) {
            QString line = rqfIn.readLine();
            if (!line.startsWith("PyQt6"))
                rqfOut << line << "\n";
        }
        tmpReqFile.close();

        qCDebug(logVEnv).noquote().nospace()
            << "Creating virtualenv \"" << venvName << "\" using requirements file: " << requirementsFile;
    }

    qCDebug(logVEnv).noquote() << "Creating new Python virtualenv in:" << venvDir;
    QTextStream out(&shFile);
    out << "#!/bin/bash\n\n"
        << "run_check() {\n"
        << "    echo -e \"\\033[1;33m-\\033[0m \\033[1m$@\\033[0m\"\n"
        << "    $@\n"
        << "    if [ $? -ne 0 ]\n"
        << "    then\n"
        << "        echo \"\"\n"
        << "        read -p \"Command failed to run. Press enter to exit.\"\n"
        << "        exit 1\n"
        << "    fi\n"
        << "}\n"
        << "export PATH=$PATH:/app/bin\n\n"

        << "cd " << shellQuote(venvDir) << "\n"
        << "run_check virtualenv ."
        << "\n"
        << "run_check source " << shellQuote(QStringLiteral("%1/bin/activate").arg(venvDir)) << "\n"
        << "run_check pip install -r " << shellQuote(tmpRequirementsFname) << "\n"

        << "echo \"\"\n"
        << "read -p \"Success! Press any key to exit.\""
        << "\n";
    shFile.flush();
    shFile.setPermissions(QFileDevice::ExeUser | QFileDevice::ReadUser | QFileDevice::WriteUser);
    shFile.close();

    int ret = runInTerminal(
        tmpCommandFile, QStringList(), venvDir, QStringLiteral("Creating virtual Python environment"));

    shFile.remove();
    tmpReqFile.remove();
    if (ret == 0) {
        if (!injectSystemPyQtBindings(venvDir))
            return std::unexpected("Unable to inject system PyQt bindings into virtualenv");

        if (!requirementsFile.isEmpty()) {
            const auto b3sumResult = blake3HashForFile(requirementsFile);

            if (!b3sumResult.has_value())
                return std::unexpected(
                    QStringLiteral("Unable to compute requirements checksum: %1").arg(b3sumResult.error()));

            if (!writeRequirementsHashMetadata(venvDir, b3sumResult.value()))
                qCWarning(logVEnv).noquote() << "Unable to persist requirements metadata for virtualenv:" << venvName;
        }

        return venvDir;
    }

    // failure, let's try to remove the bad virtualenv (failures to do so are ignored)
    QDir(venvDir).removeRecursively();

    return std::unexpected(
        QStringLiteral("Environment creation failed - refer to the terminal log for more information."));
}

} // namespace Syntalos

Updated on 2026-03-30 at 00:43:15 +0000