WIP: improve coefficient readout from LibreCAL

Software/PC_Application/LibreVNA-GUI/Calibration/LibreCAL/caldevice.cpp

@@ -1,7 +1,5 @@
 #include "caldevice.h"
 
-#include <thread>
-
 #include <QDebug>
 #include <QDateTime>
 using namespace std;
@@ -33,6 +31,8 @@ static QString getLocalDateTimeWithUtcOffset()
 CalDevice::CalDevice(QString serial) :
     usb(new USBDevice(serial))
 {
+    loadThread = nullptr;
+
     // Check device identification
     auto id = usb->Query("*IDN?");
     if(!id.startsWith("LibreCAL,")) {
@@ -169,10 +169,24 @@ QString CalDevice::getDateTimeUTC()
     }
 }
 
-void CalDevice::loadCoefficientSets(QStringList names)
+void CalDevice::loadCoefficientSets(QStringList names, bool fast)
 {
     coeffSets.clear();
-    new std::thread(&CalDevice::loadCoefficientSetsThread, this, names);
+    abortLoading = false;
+    if(fast) {
+        loadThread = new std::thread(&CalDevice::loadCoefficientSetsThreadFast, this, names);
+    } else {
+        loadThread = new std::thread(&CalDevice::loadCoefficientSetsThreadSlow, this, names);
+    }
+}
+
+void CalDevice::abortCoefficientLoading()
+{
+    if(loadThread) {
+        abortLoading = true;
+        loadThread->join();
+        loadThread = nullptr;
+    }
 }
 
 void CalDevice::saveCoefficientSets()
@@ -185,7 +199,7 @@ void CalDevice::saveCoefficientSets()
     }
 }
 
-void CalDevice::loadCoefficientSetsThread(QStringList names)
+void CalDevice::loadCoefficientSetsThreadSlow(QStringList names)
 {
     QStringList coeffList = getCoefficientSetNames();
     if(coeffList.empty()) {
@@ -208,6 +222,9 @@ void CalDevice::loadCoefficientSetsThread(QStringList names)
     unsigned long totalPoints = 0;
     for(auto name : coeffList) {
         for(int i=1;i<=numPorts;i++) {
+            if(abortLoading) {
+                return;
+            }
             totalPoints += usb->Query(":COEFF:NUM? "+name+" P"+QString::number(i)+"_OPEN").toInt();
             totalPoints += usb->Query(":COEFF:NUM? "+name+" P"+QString::number(i)+"_SHORT").toInt();
             totalPoints += usb->Query(":COEFF:NUM? "+name+" P"+QString::number(i)+"_LOAD").toInt();
@@ -234,6 +251,9 @@ void CalDevice::loadCoefficientSetsThread(QStringList names)
                     c->t = Touchstone(1);
                 }
                 for(int i=0;i<points;i++) {
+                    if(abortLoading) {
+                        break;
+                    }
                     QString pString = usb->Query(":COEFF:GET? "+setName+" "+paramName+" "+QString::number(i));
                     QStringList values = pString.split(",");
                     Touchstone::Datapoint p;
@@ -265,6 +285,124 @@ void CalDevice::loadCoefficientSetsThread(QStringList names)
             for(int j=i+1;j<=numPorts;j++) {
                 set.throughs.push_back(createCoefficient(name, "P"+QString::number(i)+QString::number(j)+"_THROUGH"));
             }
+            if(abortLoading) {
+                return;
+            }
+        }
+        coeffSets.push_back(set);
+    }
+    emit updateCoefficientsDone(true);
+}
+
+void CalDevice::loadCoefficientSetsThreadFast(QStringList names)
+{
+    QStringList coeffList = getCoefficientSetNames();
+    if(coeffList.empty()) {
+        // something went wrong
+        emit updateCoefficientsDone(false);
+        return;
+    }
+    if(names.size() > 0) {
+        // check if all the requested names are actually available
+        for(auto n : names) {
+            if(!coeffList.contains(n)) {
+                // this coefficient does not exist
+                emit updateCoefficientsDone(false);
+                return;
+            }
+        }
+        coeffList = names;
+    }
+
+    QStringList coeffNames;
+    for(int i=1;i<=numPorts;i++) {
+        coeffNames.append("P"+QString::number(i)+"_OPEN");
+        coeffNames.append("P"+QString::number(i)+"_SHORT");
+        coeffNames.append("P"+QString::number(i)+"_LOAD");
+        for(int j=i+1;j<=numPorts;j++) {
+            coeffNames.append("P"+QString::number(i)+QString::number(j)+"_THROUGH");
+        }
+    }
+
+    int total_coeffs = coeffNames.size() * names.size();
+    int read_coeffs = 0;
+
+    for(auto name : coeffList) {
+        // create the coefficient set
+        CoefficientSet set;
+        set.name = name;
+        set.ports = numPorts;
+
+        auto createCoefficient = [&](QString setName, QString paramName) -> CoefficientSet::Coefficient* {
+            CoefficientSet::Coefficient *c = new CoefficientSet::Coefficient();
+            // ask for the whole set at once
+            usb->send(":COEFF:GET? "+setName+" "+paramName);
+            // handle incoming lines
+            if(paramName.endsWith("THROUGH")) {
+                c->t = Touchstone(2);
+            } else {
+                c->t = Touchstone(1);
+            }
+            c->t.setFilename("LibreCAL/"+paramName);
+            while(true) {
+                QString line;
+                if(!usb->receive(&line, 100)) {
+                    // failed to receive something, abort
+                    return c;
+                }
+                if(line.startsWith("ERROR")) {
+                    // something went wront
+                    return c;
+                }
+                // ignore start, comments and option line
+                if(line.startsWith("START") || line.startsWith("!") || line.startsWith("#")) {
+                    // ignore
+                    continue;
+                }
+                if(line.startsWith("END")) {
+                    // got all data
+                    return c;
+                }
+                // parse the values
+                try {
+                    QStringList values = line.split(" ");
+                    Touchstone::Datapoint p;
+                    p.frequency = values[0].toDouble() * 1e9;
+                    for(int j = 0;j<(values.size()-1)/2;j++) {
+                        double real = values[1+j*2].toDouble();
+                        double imag = values[2+j*2].toDouble();
+                        p.S.push_back(complex<double>(real, imag));
+                    }
+                    if(p.S.size() == 4) {
+                        // S21 and S12 are swapped in the touchstone file order (S21 goes first)
+                        // but Touchstone::AddDatapoint expects S11 S12 S21 S22 order. Swap to match that
+                        swap(p.S[1], p.S[2]);
+                    }
+                    c->t.AddDatapoint(p);
+                } catch (...) {
+                    return c;
+                }
+            }
+        };
+
+        for(auto coeff : coeffNames) {
+            auto c = createCoefficient(name, coeff);
+            if(abortLoading) {
+                return;
+            }
+            if(c) {
+                if(coeff.endsWith("_OPEN")) {
+                    set.opens.push_back(c);
+                } else if(coeff.endsWith("_SHORT")) {
+                    set.shorts.push_back(c);
+                } else if(coeff.endsWith("_LOAD")) {
+                    set.loads.push_back(c);
+                } else if(coeff.endsWith("_THROUGH")) {
+                    set.throughs.push_back(c);
+                }
+            }
+            read_coeffs++;
+            emit updateCoefficientsPercent(read_coeffs * 100 / total_coeffs);
         }
         coeffSets.push_back(set);
     }

Software/PC_Application/LibreVNA-GUI/Calibration/LibreCAL/caldevice.h

@@ -6,6 +6,7 @@
 
 #include <QString>
 #include <QObject>
+#include <thread>
 
 class CalDevice : public QObject
 {
@@ -70,7 +71,9 @@ public:
     // Extracts the coefficients from the device. This is done with a dedicated thread.
     // Do not call any other functions until the update is finished. Process can be
     // monitored through the updateCoefficientsPercent and updateCoefficientsDone signals
-    void loadCoefficientSets(QStringList names = QStringList());
+    void loadCoefficientSets(QStringList names = QStringList(), bool fast=true);
+
+    void abortCoefficientLoading();
     // Writes coefficient sets to the device. This will only write modified files to save
     // time. This is done with a dedicated thread.
     // Do not call any other functions until the update is finished. Process can be
@@ -92,12 +95,15 @@ signals:
     void disconnected();
 
 private:
-    void loadCoefficientSetsThread(QStringList names = QStringList());
+    void loadCoefficientSetsThreadSlow(QStringList names = QStringList());
+    void loadCoefficientSetsThreadFast(QStringList names = QStringList());
     void saveCoefficientSetsThread();
 
     USBDevice *usb;
     QString firmware;
     int numPorts;
+    std::thread *loadThread;
+    bool abortLoading;
 
     float firmware_major_minor;
 

Software/PC_Application/LibreVNA-GUI/Calibration/LibreCAL/librecaldialog.cpp

@@ -19,6 +19,7 @@ LibreCALDialog::LibreCALDialog(Calibration *cal) :
     busy(false)
 {
     ui->setupUi(this);
+    setAttribute(Qt::WA_DeleteOnClose);
 
     createPortAssignmentUI();
 
@@ -87,11 +88,6 @@ LibreCALDialog::LibreCALDialog(Calibration *cal) :
         ui->cbDevice->addItem(device);
     }
 
-    connect(this, &QDialog::finished, [=](){
-        delete device;
-        device = nullptr;
-    });
-
     connect(ui->start, &QPushButton::clicked, this, &LibreCALDialog::startCalibration);
 
     updateCalibrationStartStatus();
@@ -102,6 +98,9 @@ LibreCALDialog::LibreCALDialog(Calibration *cal) :
 
 LibreCALDialog::~LibreCALDialog()
 {
+    if(device) {
+        device->abortCoefficientLoading();
+    }
     delete device;
     delete ui;
 }

Software/PC_Application/LibreVNA-GUI/Calibration/LibreCAL/usbdevice.cpp

@@ -21,6 +21,7 @@ static constexpr USBID IDs[] = {
 
 USBDevice::USBDevice(QString serial)
 {
+    rx_cnt = 0;
     m_handle = nullptr;
     libusb_init(&m_context);
 #if LIBUSB_API_VERSION >= 0x01000106
@@ -200,7 +201,7 @@ void USBDevice::SearchDevices(std::function<bool (libusb_device_handle *, QStrin
 
 bool USBDevice::send(const QString &s)
 {
-//    qDebug() << "Send:"<<s;
+    qDebug() << "Send:"<<s;
     unsigned char data[s.size()+2];
     memcpy(data, s.toLatin1().data(), s.size());
     memcpy(&data[s.size()], "\r\n", 2);
@@ -213,37 +214,69 @@ bool USBDevice::send(const QString &s)
     }
 }
 
-bool USBDevice::receive(QString *s)
+bool USBDevice::receive(QString *s, unsigned int timeout)
 {
-    char data[512];
+    auto checkForCompleteLine = [this, s]() -> bool {
-    memset(data, 0, sizeof(data));
+        for(unsigned int i=1;i<rx_cnt;i++) {
-    int actual;
+            if(rx_buf[i] == '\n' && rx_buf[i-1] == '\r') {
-    int rcvCnt = 0;
+                rx_buf[i-1] = '\0';
-    bool endOfLineFound = false;
+                if(s) {
-    int res;
+                    *s = QString(rx_buf);
-    do {
+//                      qDebug() << "Receive:"<<*s;
-        res = libusb_bulk_transfer(m_handle, LIBUSB_ENDPOINT_IN | 0x03, (unsigned char*) &data[rcvCnt], sizeof(data) - rcvCnt, &actual, 2000);
+                }
-        for(int i=rcvCnt;i<rcvCnt+actual;i++) {
+                memmove(rx_buf, &rx_buf[i+1], sizeof(rx_buf) - (i+1));
-            if(i == 0) {
+                rx_cnt -= i+1;
-                continue;
+                rx_buf[rx_cnt] = 0;
-            }
+//                qDebug() << "Remaining in buffer:" << QString(rx_buf);
-            if(data[i] == '\n' && data[i-1] == '\r') {
+                return true;
-                endOfLineFound = true;
-                data[i-1] = '\0';
-                break;
             }
         }
-        rcvCnt += actual;
-    } while(res == 0 && !endOfLineFound);
-    if(res == 0) {
-        if(s) {
-            *s = QString(data);
-//            qDebug() << "Receive:"<<*s;
-        }
-        return true;
-    } else {
         return false;
+    };
+
+    int res = 0;
+    if(!checkForCompleteLine()) {
+        // needs to receive data
+        int actual;
+        do {
+            res = libusb_bulk_transfer(m_handle, LIBUSB_ENDPOINT_IN | 0x03, (unsigned char*) &rx_buf[rx_cnt], 512, &actual, timeout);
+            rx_cnt += actual;
+//            qDebug() << "received" << actual << "bytes. Total in buffer:" << rx_cnt;
+            if(checkForCompleteLine()) {
+                return true;
+            }
+        } while(res == 0);
     }
+    return res == 0;
+//    char data[512];
+//    memset(data, 0, sizeof(data));
+//    int actual;
+//    int rcvCnt = 0;
+//    bool endOfLineFound = false;
+//    int res;
+//    do {
+//        res = libusb_bulk_transfer(m_handle, LIBUSB_ENDPOINT_IN | 0x03, (unsigned char*) &data[rcvCnt], sizeof(data) - rcvCnt, &actual, 2000);
+//        for(int i=rcvCnt;i<rcvCnt+actual;i++) {
+//            if(i == 0) {
+//                continue;
+//            }
+//            if(data[i] == '\n' && data[i-1] == '\r') {
+//                endOfLineFound = true;
+//                data[i-1] = '\0';
+//                break;
+//            }
+//        }
+//        rcvCnt += actual;
+//    } while(res == 0 && !endOfLineFound);
+//    if(res == 0) {
+//        if(s) {
+//            *s = QString(data);
+////            qDebug() << "Receive:"<<*s;
+//        }
+//        return true;
+//    } else {
+//        return false;
+//    }
 }
 
 QString USBDevice::serial() const

Software/PC_Application/LibreVNA-GUI/Calibration/LibreCAL/usbdevice.h

@@ -23,17 +23,20 @@ public:
     // Returns serial numbers of all connected devices
     static std::set<QString> GetDevices();
 
+    bool send(const QString &s);
+    bool receive(QString *s, unsigned int timeout = 2000);
 signals:
     void communicationFailure();
 
 private:
     static void SearchDevices(std::function<bool (libusb_device_handle *, QString)> foundCallback, libusb_context *context, bool ignoreOpenError);
-    bool send(const QString &s);
-    bool receive(QString *s);
     libusb_device_handle *m_handle;
     libusb_context *m_context;
 
     QString m_serial;
+
+    char rx_buf[1024];
+    unsigned int rx_cnt;
 };
 
 #endif // DEVICE_H