Changeset 2767 for trunk/src/Scantable.cpp
- Timestamp:
- 02/08/13 22:23:22 (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/Scantable.cpp
r2740 r2767 2490 2490 } 2491 2491 2492 void Scantable::polyBaseline(const std::vector<bool>& mask, int order, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 2493 { 2492 std::vector<std::string> Scantable::applyBaselineTable(const std::string& bltable, const std::string& outbltable, const bool outbltableexists, const bool overwrite) 2493 { 2494 STBaselineTable* btin = new STBaselineTable(bltable); 2495 2496 Vector<Bool> applyCol = btin->getApply(); 2497 int nRowBl = applyCol.size(); 2498 if (nRowBl != nrow()) { 2499 throw(AipsError("Scantable and bltable have different number of rows.")); 2500 } 2501 2502 std::vector<std::string> res; 2503 res.clear(); 2504 2505 bool outBaselineTable = ((outbltable != "") && (!outbltableexists || overwrite)); 2506 bool bltableidentical = (bltable == outbltable); 2507 STBaselineTable* btout = new STBaselineTable(*this); 2508 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 2509 Vector<Double> timeSecCol = tcol->getColumn(); 2510 2511 for (int whichrow = 0; whichrow < nRowBl; ++whichrow) { 2512 if (applyCol[whichrow]) { 2513 std::vector<float> spec = getSpectrum(whichrow); 2514 2515 //--channel mask-- maybe the following choices should be available 2516 std::vector<bool> mask = btin->getMask(whichrow); //use mask_bltable only (default) 2517 //std::vector<bool> mask = getMask(whichrow); // or mask_scantable only 2518 //std::vector<bool> mask = getCompositeChanMask(whichrow, btin->getMask(whichrow)); // or mask_scantable && mask_bltable 2519 2520 STBaselineFunc::FuncName ftype = btin->getFunctionName(whichrow); 2521 std::vector<int> fpar = btin->getFuncParam(whichrow); 2522 std::vector<float> params; 2523 float rms; 2524 std::vector<float> resfit = doApplyBaselineTable(spec, mask, ftype, fpar, params, rms); 2525 2526 setSpectrum(resfit, whichrow); 2527 2528 res.push_back(packFittingResults(whichrow, params, rms)); 2529 2530 if (outBaselineTable) { 2531 Vector<Int> fparam(fpar.size()); 2532 for (uInt j = 0; j < fparam.size(); ++j) { 2533 fparam[j] = (Int)fpar[j]; 2534 } 2535 std::vector<int> maskList0; 2536 maskList0.clear(); 2537 for (uInt j = 0; j < mask.size(); ++j) { 2538 if (mask[j]) { 2539 if ((j == 0)||(j == mask.size()-1)) { 2540 maskList0.push_back(j); 2541 } else { 2542 if ((mask[j])&&(!mask[j-1])) { 2543 maskList0.push_back(j); 2544 } 2545 if ((mask[j])&&(!mask[j+1])) { 2546 maskList0.push_back(j); 2547 } 2548 } 2549 } 2550 } 2551 Vector<uInt> maskList(maskList0.size()); 2552 for (uInt j = 0; j < maskList0.size(); ++j) { 2553 maskList[j] = (uInt)maskList0[j]; 2554 } 2555 2556 if (outbltableexists) { 2557 if (overwrite) { 2558 if (bltableidentical) { 2559 btin->setresult(uInt(whichrow), Vector<Float>(params), Float(rms)); 2560 } else { 2561 btout->setresult(uInt(whichrow), Vector<Float>(params), Float(rms)); 2562 } 2563 } 2564 } else { 2565 btout->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), 2566 uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 2567 uInt(0), timeSecCol[whichrow], Bool(true), ftype, fparam, 2568 Vector<Float>(), maskList, Vector<Float>(params), 2569 Float(rms), uInt(spec.size()), Float(3.0), uInt(0), 2570 Float(0.0), uInt(0), Vector<uInt>()); 2571 } 2572 } 2573 } 2574 } 2575 2576 if (outBaselineTable) { 2577 if (bltableidentical) { 2578 btin->save(outbltable); 2579 } else { 2580 btout->save(outbltable); 2581 } 2582 } 2583 delete btin; 2584 delete btout; 2585 2586 return res; 2587 } 2588 2589 std::vector<std::string> Scantable::subBaseline(const std::vector<std::string>& blInfoList, const std::string& outbltable, const bool outbltableexists, const bool overwrite) 2590 { 2591 int nRowBl = blInfoList.size(); 2592 int nRowSt = nrow(); 2593 2594 std::vector<std::string> res; 2595 res.clear(); 2596 2597 bool outBaselineTable = ((outbltable != "") && (!outbltableexists || overwrite)); 2598 if ((outbltable != "") && outbltableexists && !overwrite) { 2599 throw(AipsError("Cannot overwrite bltable. Set overwrite=True.")); 2600 } 2601 2602 STBaselineTable* bt = new STBaselineTable(*this); 2603 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 2604 Vector<Double> timeSecCol = tcol->getColumn(); 2605 2606 if (outBaselineTable && !outbltableexists) { 2607 for (int i = 0; i < nRowSt; ++i) { 2608 bt->appendbasedata(getScan(i), getCycle(i), getBeam(i), getIF(i), getPol(i), 2609 0, timeSecCol[i]); 2610 bt->setApply(i, false); 2611 } 2612 } 2613 2614 for (int i = 0; i < nRowBl; ++i) { 2615 int irow; 2616 STBaselineFunc::FuncName ftype; 2617 std::vector<bool> mask; 2618 std::vector<int> fpar; 2619 float clipth; 2620 int clipn; 2621 bool uself; 2622 float lfth; 2623 std::vector<int> lfedge; 2624 int lfavg; 2625 parseBlInfo(blInfoList[i], irow, ftype, fpar, mask, clipth, clipn, uself, lfth, lfedge, lfavg); 2626 2627 if (irow < nRowSt) { 2628 std::vector<float> spec = getSpectrum(irow); 2629 std::vector<float> params; 2630 float rms; 2631 std::vector<bool> finalmask; 2632 2633 std::vector<float> resfit = doSubtractBaseline(spec, mask, ftype, fpar, params, rms, finalmask, clipth, clipn, uself, irow, lfth, lfedge, lfavg); 2634 setSpectrum(resfit, irow); 2635 res.push_back(packFittingResults(irow, params, rms)); 2636 2637 if (outBaselineTable) { 2638 Vector<Int> fparam(fpar.size()); 2639 for (uInt j = 0; j < fparam.size(); ++j) { 2640 fparam[j] = (Int)fpar[j]; 2641 } 2642 std::vector<int> maskList0; 2643 maskList0.clear(); 2644 for (uInt j = 0; j < finalmask.size(); ++j) { 2645 if (finalmask[j]) { 2646 if ((j == 0)||(j == finalmask.size()-1)) { 2647 maskList0.push_back(j); 2648 } else { 2649 if ((finalmask[j])&&(!finalmask[j-1])) { 2650 maskList0.push_back(j); 2651 } 2652 if ((finalmask[j])&&(!finalmask[j+1])) { 2653 maskList0.push_back(j); 2654 } 2655 } 2656 } 2657 } 2658 Vector<uInt> maskList(maskList0.size()); 2659 for (uInt j = 0; j < maskList0.size(); ++j) { 2660 maskList[j] = (uInt)maskList0[j]; 2661 } 2662 2663 bt->setdata(uInt(irow), 2664 uInt(getScan(irow)), uInt(getCycle(irow)), 2665 uInt(getBeam(irow)), uInt(getIF(irow)), uInt(getPol(irow)), 2666 uInt(0), timeSecCol[irow], Bool(true), ftype, fparam, 2667 Vector<Float>(), maskList, Vector<Float>(params), 2668 Float(rms), uInt(spec.size()), Float(clipth), uInt(clipn), 2669 Float(0.0), uInt(0), Vector<uInt>()); 2670 } 2671 2672 } 2673 } 2674 2675 if (outBaselineTable) { 2676 bt->save(outbltable); 2677 } 2678 delete bt; 2679 2680 return res; 2681 } 2682 2683 std::vector<float> Scantable::doApplyBaselineTable(std::vector<float>& spec, std::vector<bool>& mask, const STBaselineFunc::FuncName ftype, std::vector<int>& fpar, std::vector<float>& params, float&rms) 2684 { 2685 std::vector<bool> finalmask; 2686 std::vector<int> lfedge; 2687 return doSubtractBaseline(spec, mask, ftype, fpar, params, rms, finalmask, 0.0, 0, false, 0, 0.0, lfedge, 0); 2688 } 2689 2690 std::vector<float> Scantable::doSubtractBaseline(std::vector<float>& spec, std::vector<bool>& mask, const STBaselineFunc::FuncName ftype, std::vector<int>& fpar, std::vector<float>& params, float&rms, std::vector<bool>& finalmask, float clipth, int clipn, bool uself, int irow, float lfth, std::vector<int>& lfedge, int lfavg) 2691 { 2692 //---replace mask with (mask AND mask_linefinder). 2693 if (uself) { 2694 STLineFinder lineFinder = STLineFinder(); 2695 lineFinder.setOptions(lfth, 3, lfavg); 2696 2697 int minEdgeSize = getIFNos().size()*2; 2698 int edgeSize = lfedge.size(); 2699 std::vector<int> currentEdge; 2700 currentEdge.clear(); 2701 if (edgeSize >= 2) { 2702 int idx = 0; 2703 if (edgeSize > 2) { 2704 if (edgeSize < minEdgeSize) { 2705 throw(AipsError("Length of edge element info is less than that of IFs")); 2706 } 2707 idx = 2 * getIF(irow); 2708 } 2709 currentEdge.push_back(lfedge[idx]); 2710 currentEdge.push_back(lfedge[idx+1]); 2711 } else { 2712 throw(AipsError("Wrong length of edge element")); 2713 } 2714 lineFinder.setData(spec); 2715 lineFinder.findLines(getCompositeChanMask(irow, mask), currentEdge, irow); 2716 2717 std::vector<bool> lfcompmask = lineFinder.getMask(); 2718 mask.clear(); 2719 mask.resize(lfcompmask.size()); 2720 for (uInt i = 0; i < mask.size(); ++i) { 2721 mask[i] = lfcompmask[i]; 2722 } 2723 } 2724 //--- 2725 2726 std::vector<float> res; 2727 if (ftype == STBaselineFunc::Polynomial) { 2728 res = doPolynomialFitting(spec, mask, fpar[0], params, rms, finalmask, clipth, clipn); 2729 } else if (ftype == STBaselineFunc::Chebyshev) { 2730 res = doChebyshevFitting(spec, mask, fpar[0], params, rms, finalmask, clipth, clipn); 2731 } else if (ftype == STBaselineFunc::CSpline) { 2732 if (fpar.size() > 1) { // reading from baseline table in which pieceEdges are already calculated and stored. 2733 res = doCubicSplineFitting(spec, mask, fpar, params, rms, finalmask, clipth, clipn); 2734 } else { // usual cspline fitting by giving nPiece only. fpar will be replaced with pieceEdges. 2735 res = doCubicSplineFitting(spec, mask, fpar[0], fpar, params, rms, finalmask, clipth, clipn); 2736 } 2737 } else if (ftype == STBaselineFunc::Sinusoid) { 2738 res = doSinusoidFitting(spec, mask, fpar, params, rms, finalmask, clipth, clipn); 2739 } 2740 2741 return res; 2742 } 2743 2744 std::string Scantable::packFittingResults(const int irow, const std::vector<float>& params, const float rms) 2745 { 2746 // returned value: "irow:params[0],params[1],..,params[n-1]:rms" 2747 ostringstream os; 2748 os << irow << ':'; 2749 for (uInt i = 0; i < params.size(); ++i) { 2750 if (i > 0) { 2751 os << ','; 2752 } 2753 os << params[i]; 2754 } 2755 os << ':' << rms; 2756 2757 return os.str(); 2758 } 2759 2760 void Scantable::parseBlInfo(const std::string& blInfo, int& irow, STBaselineFunc::FuncName& ftype, std::vector<int>& fpar, std::vector<bool>& mask, float& thresClip, int& nIterClip, bool& useLineFinder, float& thresLF, std::vector<int>& edgeLF, int& avgLF) 2761 { 2762 // The baseline info to be parsed must be column-delimited string like 2763 // "0:chebyshev:5:3,5,169,174,485,487" where the elements are 2764 // row number, funcType, funcOrder, maskList, clipThreshold, clipNIter, 2765 // useLineFinder, lfThreshold, lfEdge and lfChanAvgLimit. 2766 2767 std::vector<string> res = splitToStringList(blInfo, ':'); 2768 if (res.size() < 4) { 2769 throw(AipsError("baseline info has bad format")) ; 2770 } 2771 2772 string ftype0, fpar0, masklist0, uself0, edge0; 2773 std::vector<int> masklist; 2774 2775 stringstream ss; 2776 ss << res[0]; 2777 ss >> irow; 2778 ss.clear(); ss.str(""); 2779 2780 ss << res[1]; 2781 ss >> ftype0; 2782 if (ftype0 == "poly") { 2783 ftype = STBaselineFunc::Polynomial; 2784 } else if (ftype0 == "cspline") { 2785 ftype = STBaselineFunc::CSpline; 2786 } else if (ftype0 == "sinusoid") { 2787 ftype = STBaselineFunc::Sinusoid; 2788 } else if (ftype0 == "chebyshev") { 2789 ftype = STBaselineFunc::Chebyshev; 2790 } else { 2791 throw(AipsError("invalid function type.")); 2792 } 2793 ss.clear(); ss.str(""); 2794 2795 ss << res[2]; 2796 ss >> fpar0; 2797 fpar = splitToIntList(fpar0, ','); 2798 ss.clear(); ss.str(""); 2799 2800 ss << res[3]; 2801 ss >> masklist0; 2802 mask = getMaskFromMaskList(nchan(getIF(irow)), splitToIntList(masklist0, ',')); 2803 2804 ss << res[4]; 2805 ss >> thresClip; 2806 ss.clear(); ss.str(""); 2807 2808 ss << res[5]; 2809 ss >> nIterClip; 2810 ss.clear(); ss.str(""); 2811 2812 ss << res[6]; 2813 ss >> uself0; 2814 if (uself0 == "true") { 2815 useLineFinder = true; 2816 } else { 2817 useLineFinder = false; 2818 } 2819 ss.clear(); ss.str(""); 2820 2821 if (useLineFinder) { 2822 ss << res[7]; 2823 ss >> thresLF; 2824 ss.clear(); ss.str(""); 2825 2826 ss << res[8]; 2827 ss >> edge0; 2828 edgeLF = splitToIntList(edge0, ','); 2829 ss.clear(); ss.str(""); 2830 2831 ss << res[9]; 2832 ss >> avgLF; 2833 ss.clear(); ss.str(""); 2834 } 2835 2836 } 2837 2838 std::vector<int> Scantable::splitToIntList(const std::string& s, const char delim) 2839 { 2840 istringstream iss(s); 2841 string tmp; 2842 int tmpi; 2843 std::vector<int> res; 2844 stringstream ss; 2845 while (getline(iss, tmp, delim)) { 2846 ss << tmp; 2847 ss >> tmpi; 2848 res.push_back(tmpi); 2849 ss.clear(); ss.str(""); 2850 } 2851 2852 return res; 2853 } 2854 2855 std::vector<string> Scantable::splitToStringList(const std::string& s, const char delim) 2856 { 2857 istringstream iss(s); 2858 std::string tmp; 2859 std::vector<string> res; 2860 while (getline(iss, tmp, delim)) { 2861 res.push_back(tmp); 2862 } 2863 2864 return res; 2865 } 2866 2867 std::vector<bool> Scantable::getMaskFromMaskList(const int nchan, const std::vector<int>& masklist) 2868 { 2869 if (masklist.size() % 2 != 0) { 2870 throw(AipsError("masklist must have even number of elements.")); 2871 } 2872 2873 std::vector<bool> res(nchan); 2874 2875 for (int i = 0; i < nchan; ++i) { 2876 res[i] = false; 2877 } 2878 for (uInt j = 0; j < masklist.size(); j += 2) { 2879 for (int i = masklist[j]; i <= masklist[j+1]; ++i) { 2880 res[i] = true; 2881 } 2882 } 2883 2884 return res; 2885 } 2886 2887 void Scantable::polyBaseline(const std::vector<bool>& mask, int order, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 2888 { 2889 /**** 2890 double TimeStart = mathutil::gettimeofday_sec(); 2891 ****/ 2892 2494 2893 try { 2495 2894 ofstream ofs; … … 2511 2910 } 2512 2911 2513 Fitter fitter = Fitter();2514 fitter.setExpression("poly", order);2515 //fitter.setIterClipping(thresClip, nIterClip);2516 2517 int nRow = nrow();2518 std::vector<bool> chanMask;2519 2912 bool showProgress; 2520 2913 int minNRow; 2521 2914 parseProgressInfo(progressInfo, showProgress, minNRow); 2522 2915 2916 int nRow = nrow(); 2917 std::vector<bool> chanMask; 2918 std::vector<bool> finalChanMask; 2919 float rms; 2920 2921 //--- 2922 bool outBaselintTable = (bltable != ""); 2923 STBaselineTable* bt = new STBaselineTable(*this); 2924 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 2925 Vector<Double> timeSecCol = tcol->getColumn(); 2926 //--- 2927 2523 2928 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 2929 std::vector<float> sp = getSpectrum(whichrow); 2524 2930 chanMask = getCompositeChanMask(whichrow, mask); 2525 fitBaseline(chanMask, whichrow, fitter); 2526 setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 2527 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "polyBaseline()", fitter); 2931 std::vector<float> params(order+1); 2932 int nClipped = 0; 2933 std::vector<float> res = doPolynomialFitting(sp, chanMask, order, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 2934 2935 //--- 2936 if (outBaselintTable) { 2937 Vector<Int> fparam(1); 2938 fparam[0] = order; 2939 std::vector<int> finalMaskList0; 2940 finalMaskList0.clear(); 2941 for (uInt i = 0; i < finalChanMask.size(); ++i) { 2942 if (finalChanMask[i]) { 2943 if ((i == 0)||(i == finalChanMask.size()-1)) { 2944 finalMaskList0.push_back(i); 2945 } else { 2946 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 2947 finalMaskList0.push_back(i); 2948 } 2949 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 2950 finalMaskList0.push_back(i); 2951 } 2952 } 2953 } 2954 } 2955 Vector<uInt> finalMaskList(finalMaskList0.size()); 2956 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 2957 finalMaskList[i] = (uInt)finalMaskList0[i]; 2958 } 2959 2960 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 2961 uInt(0), 2962 timeSecCol[whichrow], 2963 Bool(true), 2964 STBaselineFunc::Polynomial, 2965 fparam, 2966 Vector<Float>(), 2967 finalMaskList, 2968 Vector<Float>(params), 2969 Float(rms), 2970 uInt(sp.size()), 2971 Float(thresClip), 2972 uInt(nIterClip), 2973 Float(0.0), 2974 uInt(0), 2975 Vector<uInt>()); 2976 } else { 2977 setSpectrum(res, whichrow); 2978 } 2979 //--- 2980 2981 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "chebyshevBaseline()", params, nClipped); 2528 2982 showProgressOnTerminal(whichrow, nRow, showProgress, minNRow); 2529 2983 } 2984 2985 //--- 2986 if (outBaselintTable) { 2987 bt->save(bltable); 2988 } 2989 //--- 2990 delete bt; 2530 2991 2531 2992 if (outTextFile) ofs.close(); … … 2534 2995 throw; 2535 2996 } 2536 } 2537 2538 void Scantable::autoPolyBaseline(const std::vector<bool>& mask, int order, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 2997 2998 /**** 2999 double TimeEnd = mathutil::gettimeofday_sec(); 3000 double elapse1 = TimeEnd - TimeStart; 3001 std::cout << "poly-new : " << elapse1 << " (sec.)" << endl; 3002 ****/ 3003 } 3004 3005 void Scantable::autoPolyBaseline(const std::vector<bool>& mask, int order, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 2539 3006 { 2540 3007 try { … … 2557 3024 } 2558 3025 2559 Fitter fitter = Fitter(); 2560 fitter.setExpression("poly", order); 2561 //fitter.setIterClipping(thresClip, nIterClip); 2562 2563 int nRow = nrow(); 2564 std::vector<bool> chanMask; 3026 bool showProgress; 3027 int minNRow; 3028 parseProgressInfo(progressInfo, showProgress, minNRow); 3029 2565 3030 int minEdgeSize = getIFNos().size()*2; 2566 3031 STLineFinder lineFinder = STLineFinder(); 2567 3032 lineFinder.setOptions(threshold, 3, chanAvgLimit); 2568 3033 2569 bool showProgress; 2570 int minNRow; 2571 parseProgressInfo(progressInfo, showProgress, minNRow); 3034 int nRow = nrow(); 3035 std::vector<bool> chanMask; 3036 std::vector<bool> finalChanMask; 3037 float rms; 3038 3039 //--- 3040 bool outBaselintTable = (bltable != ""); 3041 STBaselineTable* bt = new STBaselineTable(*this); 3042 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 3043 Vector<Double> timeSecCol = tcol->getColumn(); 3044 //--- 2572 3045 2573 3046 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 2574 2575 //------------------------------------------------------- 2576 //chanMask = getCompositeChanMask(whichrow, mask, edge, minEdgeSize, lineFinder); 2577 //------------------------------------------------------- 3047 std::vector<float> sp = getSpectrum(whichrow); 3048 //-- composote given channel mask and flagtra -------- 2578 3049 int edgeSize = edge.size(); 2579 3050 std::vector<int> currentEdge; 3051 currentEdge.clear(); 2580 3052 if (edgeSize >= 2) { 2581 3053 int idx = 0; … … 2591 3063 throw(AipsError("Wrong length of edge element")); 2592 3064 } 2593 lineFinder.setData( getSpectrum(whichrow));3065 lineFinder.setData(sp); 2594 3066 lineFinder.findLines(getCompositeChanMask(whichrow, mask), currentEdge, whichrow); 2595 3067 chanMask = lineFinder.getMask(); 2596 //------------------------------------------------------- 2597 2598 fitBaseline(chanMask, whichrow, fitter); 2599 setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 2600 2601 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "autoPolyBaseline()", fitter); 3068 //-- composote given channel mask and flagtra END -------- 3069 3070 std::vector<float> params(order+1); 3071 int nClipped = 0; 3072 std::vector<float> res = doPolynomialFitting(sp, chanMask, order, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 3073 3074 if (outBaselintTable) { 3075 Vector<Int> fparam(1); 3076 fparam[0] = order; 3077 std::vector<int> finalMaskList0; 3078 finalMaskList0.clear(); 3079 for (uInt i = 0; i < finalChanMask.size(); ++i) { 3080 if (finalChanMask[i]) { 3081 if ((i == 0)||(i == finalChanMask.size()-1)) { 3082 finalMaskList0.push_back(i); 3083 } else { 3084 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 3085 finalMaskList0.push_back(i); 3086 } 3087 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 3088 finalMaskList0.push_back(i); 3089 } 3090 } 3091 } 3092 } 3093 Vector<uInt> finalMaskList(finalMaskList0.size()); 3094 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 3095 finalMaskList[i] = (uInt)finalMaskList0[i]; 3096 } 3097 3098 Vector<uInt> cEdge(currentEdge.size()); 3099 for (uInt i = 0; i < currentEdge.size(); ++i) { 3100 cEdge[i] = currentEdge[i]; 3101 } 3102 3103 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 3104 uInt(0), 3105 timeSecCol[whichrow], 3106 Bool(true), 3107 STBaselineFunc::Polynomial, 3108 fparam, 3109 Vector<Float>(), 3110 finalMaskList, 3111 Vector<Float>(params), 3112 Float(rms), 3113 uInt(sp.size()), 3114 Float(thresClip), 3115 uInt(nIterClip), 3116 Float(threshold), 3117 uInt(chanAvgLimit), 3118 cEdge); 3119 } else { 3120 setSpectrum(res, whichrow); 3121 } 3122 3123 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "autoChebyshevBaseline()", params, nClipped); 2602 3124 showProgressOnTerminal(whichrow, nRow, showProgress, minNRow); 2603 3125 } 3126 3127 //--- 3128 if (outBaselintTable) { 3129 bt->save(bltable); 3130 } 3131 //--- 3132 delete bt; 2604 3133 2605 3134 if (outTextFile) ofs.close(); … … 2610 3139 } 2611 3140 2612 void Scantable::chebyshevBaseline(const std::vector<bool>& mask, int order, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 2613 { 3141 void Scantable::chebyshevBaseline(const std::vector<bool>& mask, int order, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 3142 { 3143 // 3144 double TimeStart = mathutil::gettimeofday_sec(); 3145 // 3146 2614 3147 try { 2615 3148 ofstream ofs; … … 2641 3174 2642 3175 //--- 2643 bool outBaselin eParamTable = false;3176 bool outBaselintTable = (bltable != ""); 2644 3177 STBaselineTable* bt = new STBaselineTable(*this); 3178 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 3179 Vector<Double> timeSecCol = tcol->getColumn(); 2645 3180 //--- 2646 3181 … … 2650 3185 std::vector<float> params(order+1); 2651 3186 int nClipped = 0; 2652 std::vector<float> res = doChebyshevFitting(sp, chanMask, order, params, finalChanMask, rms, nClipped, thresClip, nIterClip, getResidual);3187 std::vector<float> res = doChebyshevFitting(sp, chanMask, order, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 2653 3188 2654 3189 //--- 2655 if (outBaselineParamTable) { 3190 if (outBaselintTable) { 3191 Vector<Int> fparam(1); 3192 fparam[0] = order; 3193 std::vector<int> finalMaskList0; 3194 finalMaskList0.clear(); 3195 for (uInt i = 0; i < finalChanMask.size(); ++i) { 3196 if (finalChanMask[i]) { 3197 if ((i == 0)||(i == finalChanMask.size()-1)) { 3198 finalMaskList0.push_back(i); 3199 } else { 3200 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 3201 finalMaskList0.push_back(i); 3202 } 3203 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 3204 finalMaskList0.push_back(i); 3205 } 3206 } 3207 } 3208 } 3209 Vector<uInt> finalMaskList(finalMaskList0.size()); 3210 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 3211 finalMaskList[i] = (uInt)finalMaskList0[i]; 3212 } 3213 2656 3214 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 2657 3215 uInt(0), 2658 Double(0.0), // <-- Double(getTime(whichrow, false)),2659 uInt(nchan(whichrow)),3216 timeSecCol[whichrow], 3217 Bool(true), 2660 3218 STBaselineFunc::Chebyshev, 2661 Vector<uInt>(1), // ==> MUST BE Vector<uInt> containing 'order'. 2662 Vector<Float>(), // ==> for ffpar. ** dummy ** 3219 fparam, 3220 Vector<Float>(), 3221 finalMaskList, 3222 Vector<Float>(params), 3223 Float(rms), 3224 uInt(sp.size()), 3225 Float(thresClip), 2663 3226 uInt(nIterClip), 2664 Float(thresClip), 2665 Vector<uInt>(5), // <-- Vector<uInt>(finalChanMask), 2666 Vector<Float>(params), 2667 Float(rms)); 3227 Float(0.0), 3228 uInt(0), 3229 Vector<uInt>()); 2668 3230 } else { 2669 3231 setSpectrum(res, whichrow); … … 2676 3238 2677 3239 //--- 2678 if (outBaselin eParamTable) {2679 bt->save( "chebyparamtable");3240 if (outBaselintTable) { 3241 bt->save(bltable); 2680 3242 } 2681 3243 //--- … … 2687 3249 throw; 2688 3250 } 3251 3252 double TimeEnd = mathutil::gettimeofday_sec(); 3253 double elapse1 = TimeEnd - TimeStart; 3254 std::cout << "cheby : " << elapse1 << " (sec.)" << endl; 3255 } 3256 3257 void Scantable::autoChebyshevBaseline(const std::vector<bool>& mask, int order, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 3258 { 3259 try { 3260 ofstream ofs; 3261 String coordInfo = ""; 3262 bool hasSameNchan = true; 3263 bool outTextFile = false; 3264 bool csvFormat = false; 3265 3266 if (blfile != "") { 3267 csvFormat = (blfile.substr(0, 1) == "T"); 3268 ofs.open(blfile.substr(1).c_str(), ios::out | ios::app); 3269 if (ofs) outTextFile = true; 3270 } 3271 3272 if (outLogger || outTextFile) { 3273 coordInfo = getCoordInfo()[0]; 3274 if (coordInfo == "") coordInfo = "channel"; 3275 hasSameNchan = hasSameNchanOverIFs(); 3276 } 3277 3278 bool showProgress; 3279 int minNRow; 3280 parseProgressInfo(progressInfo, showProgress, minNRow); 3281 3282 int minEdgeSize = getIFNos().size()*2; 3283 STLineFinder lineFinder = STLineFinder(); 3284 lineFinder.setOptions(threshold, 3, chanAvgLimit); 3285 3286 int nRow = nrow(); 3287 std::vector<bool> chanMask; 3288 std::vector<bool> finalChanMask; 3289 float rms; 3290 3291 //--- 3292 bool outBaselintTable = (bltable != ""); 3293 STBaselineTable* bt = new STBaselineTable(*this); 3294 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 3295 Vector<Double> timeSecCol = tcol->getColumn(); 3296 //--- 3297 3298 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 3299 std::vector<float> sp = getSpectrum(whichrow); 3300 //-- composote given channel mask and flagtra -------- 3301 int edgeSize = edge.size(); 3302 std::vector<int> currentEdge; 3303 currentEdge.clear(); 3304 if (edgeSize >= 2) { 3305 int idx = 0; 3306 if (edgeSize > 2) { 3307 if (edgeSize < minEdgeSize) { 3308 throw(AipsError("Length of edge element info is less than that of IFs")); 3309 } 3310 idx = 2 * getIF(whichrow); 3311 } 3312 currentEdge.push_back(edge[idx]); 3313 currentEdge.push_back(edge[idx+1]); 3314 } else { 3315 throw(AipsError("Wrong length of edge element")); 3316 } 3317 lineFinder.setData(sp); 3318 lineFinder.findLines(getCompositeChanMask(whichrow, mask), currentEdge, whichrow); 3319 chanMask = lineFinder.getMask(); 3320 //-- composote given channel mask and flagtra END -------- 3321 3322 std::vector<float> params(order+1); 3323 int nClipped = 0; 3324 std::vector<float> res = doChebyshevFitting(sp, chanMask, order, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 3325 3326 if (outBaselintTable) { 3327 Vector<Int> fparam(1); 3328 fparam[0] = order; 3329 std::vector<int> finalMaskList0; 3330 finalMaskList0.clear(); 3331 for (uInt i = 0; i < finalChanMask.size(); ++i) { 3332 if (finalChanMask[i]) { 3333 if ((i == 0)||(i == finalChanMask.size()-1)) { 3334 finalMaskList0.push_back(i); 3335 } else { 3336 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 3337 finalMaskList0.push_back(i); 3338 } 3339 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 3340 finalMaskList0.push_back(i); 3341 } 3342 } 3343 } 3344 } 3345 Vector<uInt> finalMaskList(finalMaskList0.size()); 3346 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 3347 finalMaskList[i] = (uInt)finalMaskList0[i]; 3348 } 3349 3350 Vector<uInt> cEdge(currentEdge.size()); 3351 for (uInt i = 0; i < currentEdge.size(); ++i) { 3352 cEdge[i] = currentEdge[i]; 3353 } 3354 3355 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 3356 uInt(0), 3357 timeSecCol[whichrow], 3358 Bool(true), 3359 STBaselineFunc::Chebyshev, 3360 fparam, 3361 Vector<Float>(), 3362 finalMaskList, 3363 Vector<Float>(params), 3364 Float(rms), 3365 uInt(sp.size()), 3366 Float(thresClip), 3367 uInt(nIterClip), 3368 Float(threshold), 3369 uInt(chanAvgLimit), 3370 cEdge); 3371 } else { 3372 setSpectrum(res, whichrow); 3373 } 3374 3375 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "autoChebyshevBaseline()", params, nClipped); 3376 showProgressOnTerminal(whichrow, nRow, showProgress, minNRow); 3377 } 3378 3379 //--- 3380 if (outBaselintTable) { 3381 bt->save(bltable); 3382 } 3383 //--- 3384 delete bt; 3385 3386 if (outTextFile) ofs.close(); 3387 3388 } catch (...) { 3389 throw; 3390 } 2689 3391 } 2690 3392 2691 3393 double Scantable::calculateModelSelectionCriteria(const std::string& valname, const std::string& blfunc, int order, const std::vector<bool>& inMask, int whichrow, bool useLineFinder, const std::vector<int>& edge, float threshold, int chanAvgLimit) 2692 3394 { 3395 std::vector<float> sp = getSpectrum(whichrow); 3396 std::vector<bool> chanMask; 3397 chanMask.clear(); 3398 2693 3399 if (useLineFinder) { 2694 3400 int minEdgeSize = getIFNos().size()*2; 2695 3401 2696 2697 3402 int edgeSize = edge.size(); 2698 3403 std::vector<int> currentEdge; 3404 currentEdge.clear(); 2699 3405 if (edgeSize >= 2) { 2700 3406 int idx = 0; … … 2712 3418 2713 3419 STLineFinder lineFinder = STLineFinder(); 2714 std::vector<float> sp = getSpectrum(whichrow);2715 3420 lineFinder.setData(sp); 2716 3421 lineFinder.setOptions(threshold, 3, chanAvgLimit); 2717 3422 lineFinder.findLines(getCompositeChanMask(whichrow, inMask), currentEdge, whichrow); 2718 std::vector<bool> chanMask = lineFinder.getMask(); 2719 2720 return doCalculateModelSelectionCriteria(valname, sp, chanMask, blfunc, order); 3423 chanMask = lineFinder.getMask(); 2721 3424 2722 3425 } else { 2723 return doCalculateModelSelectionCriteria(valname, getSpectrum(whichrow), getCompositeChanMask(whichrow, inMask), blfunc, order); 2724 } 2725 3426 3427 chanMask = getCompositeChanMask(whichrow, inMask); 3428 } 3429 3430 return doCalculateModelSelectionCriteria(valname, sp, chanMask, blfunc, order); 2726 3431 } 2727 3432 … … 2733 3438 float rms; 2734 3439 int nClipped = 0; 2735 float thresClip = 0.0;2736 int nIterClip = 0;2737 3440 std::vector<float> res; 2738 if (blfunc == " chebyshev") {3441 if (blfunc == "poly") { 2739 3442 nparam = order + 1; 2740 res = doChebyshevFitting(spec, mask, order, params, finalChanMask, rms, nClipped, thresClip, nIterClip, true); 3443 res = doPolynomialFitting(spec, mask, order, params, rms, finalChanMask, nClipped); 3444 } else if (blfunc == "chebyshev") { 3445 nparam = order + 1; 3446 res = doChebyshevFitting(spec, mask, order, params, rms, finalChanMask, nClipped); 3447 } else if (blfunc == "cspline") { 3448 std::vector<int> pieceEdges;//(order+1); //order = npiece 3449 nparam = order + 3; 3450 //params.resize(4*order); 3451 res = doCubicSplineFitting(spec, mask, order, false, pieceEdges, params, rms, finalChanMask, nClipped); 2741 3452 } else if (blfunc == "sinusoid") { 2742 3453 std::vector<int> nWaves; … … 2746 3457 } 2747 3458 nparam = 2*order + 1; // order = nwave 2748 res = doSinusoidFitting(spec, mask, nWaves, params, nClipped, thresClip, nIterClip, true); 2749 } else if (blfunc == "cspline") { 2750 std::vector<int> pieceEdges(order+1); //order = npiece 2751 nparam = order + 3; 2752 params.resize(4*order); 2753 res = doCubicSplineFitting(spec, mask, order, pieceEdges, params, nClipped, thresClip, nIterClip, true); 3459 res = doSinusoidFitting(spec, mask, nWaves, params, rms, finalChanMask, nClipped); 2754 3460 } else { 2755 throw(AipsError("blfunc must be chebyshev, cspline or sinusoid."));3461 throw(AipsError("blfunc must be poly, chebyshev, cspline or sinusoid.")); 2756 3462 } 2757 3463 … … 2777 3483 double aic = nusedchan * (log(2.0 * PI * msq) + 1.0) + 2.0 * nparam; 2778 3484 2779 // Corrected AIC by Sugiura(1978) 3485 // Corrected AIC by Sugiura(1978) (AICc) 2780 3486 if (valname == "aicc") { 2781 3487 if (nusedchan - nparam - 1 <= 0) { … … 2803 3509 } 2804 3510 2805 void Scantable::autoChebyshevBaseline(const std::vector<bool>& mask, int order, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 2806 { 2807 try { 2808 ofstream ofs; 2809 String coordInfo = ""; 2810 bool hasSameNchan = true; 2811 bool outTextFile = false; 2812 bool csvFormat = false; 2813 2814 if (blfile != "") { 2815 csvFormat = (blfile.substr(0, 1) == "T"); 2816 ofs.open(blfile.substr(1).c_str(), ios::out | ios::app); 2817 if (ofs) outTextFile = true; 2818 } 2819 2820 if (outLogger || outTextFile) { 2821 coordInfo = getCoordInfo()[0]; 2822 if (coordInfo == "") coordInfo = "channel"; 2823 hasSameNchan = hasSameNchanOverIFs(); 2824 } 2825 2826 int nRow = nrow(); 2827 std::vector<bool> chanMask; 2828 int minEdgeSize = getIFNos().size()*2; 2829 STLineFinder lineFinder = STLineFinder(); 2830 lineFinder.setOptions(threshold, 3, chanAvgLimit); 2831 2832 bool showProgress; 2833 int minNRow; 2834 parseProgressInfo(progressInfo, showProgress, minNRow); 2835 2836 std::vector<bool> finalChanMask; 2837 float rms; 2838 2839 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 2840 std::vector<float> sp = getSpectrum(whichrow); 2841 2842 //------------------------------------------------------- 2843 //chanMask = getCompositeChanMask(whichrow, mask, edge, minEdgeSize, lineFinder); 2844 //------------------------------------------------------- 2845 int edgeSize = edge.size(); 2846 std::vector<int> currentEdge; 2847 if (edgeSize >= 2) { 2848 int idx = 0; 2849 if (edgeSize > 2) { 2850 if (edgeSize < minEdgeSize) { 2851 throw(AipsError("Length of edge element info is less than that of IFs")); 2852 } 2853 idx = 2 * getIF(whichrow); 2854 } 2855 currentEdge.push_back(edge[idx]); 2856 currentEdge.push_back(edge[idx+1]); 2857 } else { 2858 throw(AipsError("Wrong length of edge element")); 2859 } 2860 //lineFinder.setData(getSpectrum(whichrow)); 2861 lineFinder.setData(sp); 2862 lineFinder.findLines(getCompositeChanMask(whichrow, mask), currentEdge, whichrow); 2863 chanMask = lineFinder.getMask(); 2864 //------------------------------------------------------- 2865 2866 2867 //fitBaseline(chanMask, whichrow, fitter); 2868 //setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 2869 std::vector<float> params(order+1); 2870 int nClipped = 0; 2871 std::vector<float> res = doChebyshevFitting(sp, chanMask, order, params, finalChanMask, rms, nClipped, thresClip, nIterClip, getResidual); 2872 setSpectrum(res, whichrow); 2873 2874 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "autoChebyshevBaseline()", params, nClipped); 2875 showProgressOnTerminal(whichrow, nRow, showProgress, minNRow); 2876 } 2877 2878 if (outTextFile) ofs.close(); 2879 2880 } catch (...) { 2881 throw; 2882 } 2883 } 2884 2885 /* 2886 double Scantable::getChebyshevPolynomial(int n, double x) { 2887 if ((x < -1.0)||(x > 1.0)) { 2888 throw(AipsError("out of definition range (-1 <= x <= 1).")); 2889 } else if (n < 0) { 2890 throw(AipsError("the order must be zero or positive.")); 2891 } else if (n == 0) { 3511 double Scantable::getNormalPolynomial(int n, double x) { 3512 if (n == 0) { 2892 3513 return 1.0; 2893 } else if (n == 1) { 2894 return x; 3514 } else if (n > 0) { 3515 double res = 1.0; 3516 for (int i = 0; i < n; ++i) { 3517 res *= x; 3518 } 3519 return res; 2895 3520 } else { 2896 return 2.0*x*getChebyshevPolynomial(n-1, x) - getChebyshevPolynomial(n-2, x); 2897 } 2898 } 2899 */ 3521 if (x == 0.0) { 3522 throw(AipsError("infinity result: x=0 given for negative power.")); 3523 } else { 3524 return pow(x, (double)n); 3525 } 3526 } 3527 } 3528 2900 3529 double Scantable::getChebyshevPolynomial(int n, double x) { 2901 3530 if ((x < -1.0)||(x > 1.0)) { … … 2939 3568 } 2940 3569 2941 std::vector<float> Scantable::doChebyshevFitting(const std::vector<float>& data, const std::vector<bool>& mask, int order, std::vector<float>& params, std::vector<bool>& finalMask, float& rms, int& nClipped, float thresClip, int nIterClip, bool getResidual) 3570 std::vector<float> Scantable::doPolynomialFitting(const std::vector<float>& data, const std::vector<bool>& mask, int order, std::vector<float>& params, float& rms, std::vector<bool>& finalmask, float clipth, int clipn) 3571 { 3572 int nClipped = 0; 3573 return doPolynomialFitting(data, mask, order, params, rms, finalmask, nClipped, clipth, clipn); 3574 } 3575 3576 std::vector<float> Scantable::doPolynomialFitting(const std::vector<float>& data, const std::vector<bool>& mask, int order, std::vector<float>& params, float& rms, std::vector<bool>& finalMask, int& nClipped, float thresClip, int nIterClip, bool getResidual) 3577 { 3578 return doLeastSquareFitting(data, mask, &Scantable::getNormalPolynomial, order, params, rms, finalMask, nClipped, thresClip, nIterClip, getResidual); 3579 } 3580 3581 std::vector<float> Scantable::doChebyshevFitting(const std::vector<float>& data, const std::vector<bool>& mask, int order, std::vector<float>& params, float& rms, std::vector<bool>& finalmask, float clipth, int clipn) 3582 { 3583 int nClipped = 0; 3584 return doChebyshevFitting(data, mask, order, params, rms, finalmask, nClipped, clipth, clipn); 3585 } 3586 3587 std::vector<float> Scantable::doChebyshevFitting(const std::vector<float>& data, const std::vector<bool>& mask, int order, std::vector<float>& params, float& rms, std::vector<bool>& finalMask, int& nClipped, float thresClip, int nIterClip, bool getResidual) 3588 { 3589 return doLeastSquareFitting(data, mask, &Scantable::getChebyshevPolynomial, order, params, rms, finalMask, nClipped, thresClip, nIterClip, getResidual); 3590 } 3591 3592 std::vector<float> Scantable::doLeastSquareFitting(const std::vector<float>& data, const std::vector<bool>& mask, double (Scantable::*pfunc)(int, double), int order, std::vector<float>& params, float& rms, std::vector<bool>& finalMask, int& nClipped, float thresClip, int nIterClip, bool getResidual) 2942 3593 { 2943 3594 if (data.size() != mask.size()) { … … 2945 3596 } 2946 3597 if (order < 0) { 2947 throw(AipsError("maximum order of Chebyshevpolynomial must not be negative."));3598 throw(AipsError("maximum order of polynomial must not be negative.")); 2948 3599 } 2949 3600 … … 2971 3622 // xArray.size() is nDOF and xArray[*].size() is nChan. 2972 3623 // Each xArray element are as follows: 3624 // 3625 // (for normal polynomials) 3626 // xArray[0] = {1.0, 1.0, 1.0, ..., 1.0}, 3627 // xArray[1] = {0.0, 1.0, 2.0, ..., (nChan-1)} 3628 // xArray[n-1] = ..., 3629 // xArray[n] = {0.0^n, 1.0^n, 2.0^n, ..., (nChan-1)^n} 3630 // where (0 <= n <= order) 3631 // 3632 // (for Chebyshev polynomials) 2973 3633 // xArray[0] = {T0(-1), T0(2/(nChan-1)-1), T0(4/(nChan-1)-1), ..., T0(1)}, 2974 3634 // xArray[n-1] = ..., … … 2976 3636 // where (0 <= n <= order), 2977 3637 std::vector<std::vector<double> > xArray; 3638 double xFactor, xShift; 3639 if (pfunc == &Scantable::getChebyshevPolynomial) { 3640 xFactor = 2.0/(double)(nChan - 1); 3641 xShift = -1.0; 3642 } else { 3643 xFactor = 1.0; 3644 xShift = 0.0; 3645 } 2978 3646 for (int i = 0; i < nDOF; ++i) { 2979 double xFactor = 2.0/(double)(nChan - 1);2980 3647 std::vector<double> xs; 2981 3648 xs.clear(); 2982 3649 for (int j = 0; j < nChan; ++j) { 2983 xs.push_back( getChebyshevPolynomial(i, xFactor*(double)j-1.0));3650 xs.push_back((this->*pfunc)(i, xFactor*(double)j + xShift)); 2984 3651 } 2985 3652 xArray.push_back(xs); … … 3063 3730 } 3064 3731 } 3065 //compute a vector y which consists of the coefficients of the sinusoids forming the 3066 //best-fit curves (a0,s1,c1,s2,c2,...), where a0 is constant and s* and c* are of sine 3067 //and cosine functions, respectively. 3732 //compute a vector y which consists of the coefficients of Chebyshev polynomials. 3068 3733 std::vector<double> y; 3069 3734 params.clear(); … … 3118 3783 3119 3784 std::vector<float> result; 3785 result.clear(); 3120 3786 if (getResidual) { 3121 3787 for (int i = 0; i < nChan; ++i) { … … 3131 3797 } 3132 3798 3133 void Scantable::cubicSplineBaseline(const std::vector<bool>& mask, int nPiece, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 3134 { 3135 /************* 3136 double totTimeStart, totTimeEnd, blTimeStart, blTimeEnd, ioTimeStart, ioTimeEnd, msTimeStart, msTimeEnd, seTimeStart, seTimeEnd, otTimeStart, otTimeEnd, prTimeStart, prTimeEnd; 3137 double elapseMs = 0.0; 3138 double elapseSe = 0.0; 3139 double elapseOt = 0.0; 3140 double elapsePr = 0.0; 3141 double elapseBl = 0.0; 3142 double elapseIo = 0.0; 3143 totTimeStart = mathutil::gettimeofday_sec(); 3144 *************/ 3145 3799 void Scantable::cubicSplineBaseline(const std::vector<bool>& mask, int nPiece, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 3800 { 3146 3801 try { 3147 3802 ofstream ofs; … … 3163 3818 } 3164 3819 3165 //Fitter fitter = Fitter();3166 //fitter.setExpression("cspline", nPiece);3167 //fitter.setIterClipping(thresClip, nIterClip);3168 3169 3820 bool showProgress; 3170 3821 int minNRow; … … 3173 3824 int nRow = nrow(); 3174 3825 std::vector<bool> chanMask; 3175 3176 //-------------------------------- 3826 std::vector<bool> finalChanMask; 3827 float rms; 3828 3829 //--- 3830 bool outBaselintTable = (bltable != ""); 3831 STBaselineTable* bt = new STBaselineTable(*this); 3832 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 3833 Vector<Double> timeSecCol = tcol->getColumn(); 3834 //--- 3835 3836 3177 3837 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 3178 /******************3179 ioTimeStart = mathutil::gettimeofday_sec();3180 **/3181 3838 std::vector<float> sp = getSpectrum(whichrow); 3182 /**3183 ioTimeEnd = mathutil::gettimeofday_sec();3184 elapseIo += (double)(ioTimeEnd - ioTimeStart);3185 msTimeStart = mathutil::gettimeofday_sec();3186 ******************/3187 3188 3839 chanMask = getCompositeChanMask(whichrow, mask); 3189 3190 /** 3191 msTimeEnd = mathutil::gettimeofday_sec(); 3192 elapseMs += (double)(msTimeEnd - msTimeStart); 3193 blTimeStart = mathutil::gettimeofday_sec(); 3194 **/ 3195 3196 //fitBaseline(chanMask, whichrow, fitter); 3197 //setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 3198 std::vector<int> pieceEdges(nPiece+1); 3199 std::vector<float> params(nPiece*4); 3840 std::vector<int> pieceEdges;//(nPiece+1); 3841 std::vector<float> params;//(nPiece*4); 3200 3842 int nClipped = 0; 3201 std::vector<float> res = doCubicSplineFitting(sp, chanMask, nPiece, pieceEdges, params, nClipped, thresClip, nIterClip, getResidual); 3202 3203 /** 3204 blTimeEnd = mathutil::gettimeofday_sec(); 3205 elapseBl += (double)(blTimeEnd - blTimeStart); 3206 seTimeStart = mathutil::gettimeofday_sec(); 3207 **/ 3208 3209 3210 setSpectrum(res, whichrow); 3211 // 3212 3213 /** 3214 seTimeEnd = mathutil::gettimeofday_sec(); 3215 elapseSe += (double)(seTimeEnd - seTimeStart); 3216 otTimeStart = mathutil::gettimeofday_sec(); 3217 **/ 3843 std::vector<float> res = doCubicSplineFitting(sp, chanMask, nPiece, false, pieceEdges, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 3844 3845 //--- 3846 if (outBaselintTable) { 3847 Vector<Int> fparam(nPiece); 3848 for (uInt i = 0; i < fparam.size(); ++i) { 3849 fparam[i] = pieceEdges[i]; 3850 } 3851 std::vector<int> finalMaskList0; 3852 finalMaskList0.clear(); 3853 for (uInt i = 0; i < finalChanMask.size(); ++i) { 3854 if (finalChanMask[i]) { 3855 if ((i == 0)||(i == finalChanMask.size()-1)) { 3856 finalMaskList0.push_back(i); 3857 } else { 3858 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 3859 finalMaskList0.push_back(i); 3860 } 3861 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 3862 finalMaskList0.push_back(i); 3863 } 3864 } 3865 } 3866 } 3867 Vector<uInt> finalMaskList(finalMaskList0.size()); 3868 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 3869 finalMaskList[i] = (uInt)finalMaskList0[i]; 3870 } 3871 3872 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 3873 uInt(0), 3874 timeSecCol[whichrow], 3875 Bool(true), 3876 STBaselineFunc::CSpline, 3877 fparam, 3878 Vector<Float>(), 3879 finalMaskList, 3880 Vector<Float>(params), 3881 Float(rms), 3882 uInt(sp.size()), 3883 Float(thresClip), 3884 uInt(nIterClip), 3885 Float(0.0), 3886 uInt(0), 3887 Vector<uInt>()); 3888 } else { 3889 setSpectrum(res, whichrow); 3890 } 3891 //--- 3218 3892 3219 3893 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "cubicSplineBaseline()", pieceEdges, params, nClipped); 3220 3221 /**3222 otTimeEnd = mathutil::gettimeofday_sec();3223 elapseOt += (double)(otTimeEnd - otTimeStart);3224 prTimeStart = mathutil::gettimeofday_sec();3225 **/3226 3227 3894 showProgressOnTerminal(whichrow, nRow, showProgress, minNRow); 3228 3229 /****************** 3230 prTimeEnd = mathutil::gettimeofday_sec(); 3231 elapsePr += (double)(prTimeEnd - prTimeStart); 3232 ******************/ 3233 } 3234 //-------------------------------- 3895 } 3235 3896 3897 //--- 3898 if (outBaselintTable) { 3899 bt->save(bltable); 3900 } 3901 //--- 3902 delete bt; 3903 3236 3904 if (outTextFile) ofs.close(); 3237 3905 … … 3239 3907 throw; 3240 3908 } 3241 /*************** 3242 totTimeEnd = mathutil::gettimeofday_sec(); 3243 std::cout << "io : " << elapseIo << " (sec.)" << endl; 3244 std::cout << "ms : " << elapseMs << " (sec.)" << endl; 3245 std::cout << "bl : " << elapseBl << " (sec.)" << endl; 3246 std::cout << "se : " << elapseSe << " (sec.)" << endl; 3247 std::cout << "ot : " << elapseOt << " (sec.)" << endl; 3248 std::cout << "pr : " << elapsePr << " (sec.)" << endl; 3249 std::cout << "total : " << (double)(totTimeEnd - totTimeStart) << " (sec.)" << endl; 3250 ***************/ 3251 } 3252 3253 void Scantable::autoCubicSplineBaseline(const std::vector<bool>& mask, int nPiece, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 3909 } 3910 3911 void Scantable::autoCubicSplineBaseline(const std::vector<bool>& mask, int nPiece, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 3254 3912 { 3255 3913 try { … … 3272 3930 } 3273 3931 3274 //Fitter fitter = Fitter(); 3275 //fitter.setExpression("cspline", nPiece); 3276 //fitter.setIterClipping(thresClip, nIterClip); 3277 3278 int nRow = nrow(); 3279 std::vector<bool> chanMask; 3932 bool showProgress; 3933 int minNRow; 3934 parseProgressInfo(progressInfo, showProgress, minNRow); 3935 3280 3936 int minEdgeSize = getIFNos().size()*2; 3281 3937 STLineFinder lineFinder = STLineFinder(); 3282 3938 lineFinder.setOptions(threshold, 3, chanAvgLimit); 3283 3939 3284 bool showProgress; 3285 int minNRow; 3286 parseProgressInfo(progressInfo, showProgress, minNRow); 3940 int nRow = nrow(); 3941 std::vector<bool> chanMask; 3942 std::vector<bool> finalChanMask; 3943 float rms; 3944 3945 //--- 3946 bool outBaselintTable = (bltable != ""); 3947 STBaselineTable* bt = new STBaselineTable(*this); 3948 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 3949 Vector<Double> timeSecCol = tcol->getColumn(); 3950 //--- 3287 3951 3288 3952 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 3289 3953 std::vector<float> sp = getSpectrum(whichrow); 3290 3291 //------------------------------------------------------- 3292 //chanMask = getCompositeChanMask(whichrow, mask, edge, minEdgeSize, lineFinder); 3293 //------------------------------------------------------- 3954 //-- composote given channel mask and flagtra -------- 3294 3955 int edgeSize = edge.size(); 3295 3956 std::vector<int> currentEdge; 3957 currentEdge.clear(); 3296 3958 if (edgeSize >= 2) { 3297 3959 int idx = 0; … … 3307 3969 throw(AipsError("Wrong length of edge element")); 3308 3970 } 3309 //lineFinder.setData(getSpectrum(whichrow));3310 3971 lineFinder.setData(sp); 3311 3972 lineFinder.findLines(getCompositeChanMask(whichrow, mask), currentEdge, whichrow); 3312 3973 chanMask = lineFinder.getMask(); 3313 //------------------------------------------------------- 3314 3315 3316 //fitBaseline(chanMask, whichrow, fitter); 3317 //setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 3318 std::vector<int> pieceEdges(nPiece+1); 3319 std::vector<float> params(nPiece*4); 3974 //-- composote given channel mask and flagtra END -------- 3975 3976 std::vector<int> pieceEdges;//(nPiece+1); 3977 std::vector<float> params;//(nPiece*4); 3320 3978 int nClipped = 0; 3321 std::vector<float> res = doCubicSplineFitting(sp, chanMask, nPiece, pieceEdges, params, nClipped, thresClip, nIterClip, getResidual); 3322 setSpectrum(res, whichrow); 3323 // 3979 std::vector<float> res = doCubicSplineFitting(sp, chanMask, nPiece, false, pieceEdges, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 3980 3981 if (outBaselintTable) { 3982 Vector<Int> fparam(nPiece); 3983 for (uInt i = 0; i < fparam.size(); ++i) { 3984 fparam[i] = pieceEdges[i]; 3985 } 3986 std::vector<int> finalMaskList0; 3987 finalMaskList0.clear(); 3988 for (uInt i = 0; i < finalChanMask.size(); ++i) { 3989 if (finalChanMask[i]) { 3990 if ((i == 0)||(i == finalChanMask.size()-1)) { 3991 finalMaskList0.push_back(i); 3992 } else { 3993 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 3994 finalMaskList0.push_back(i); 3995 } 3996 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 3997 finalMaskList0.push_back(i); 3998 } 3999 } 4000 } 4001 } 4002 Vector<uInt> finalMaskList(finalMaskList0.size()); 4003 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 4004 finalMaskList[i] = (uInt)finalMaskList0[i]; 4005 } 4006 4007 Vector<uInt> cEdge(currentEdge.size()); 4008 for (uInt i = 0; i < currentEdge.size(); ++i) { 4009 cEdge[i] = currentEdge[i]; 4010 } 4011 4012 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 4013 uInt(0), 4014 timeSecCol[whichrow], 4015 Bool(true), 4016 STBaselineFunc::CSpline, 4017 fparam, 4018 Vector<Float>(), 4019 finalMaskList, 4020 Vector<Float>(params), 4021 Float(rms), 4022 uInt(sp.size()), 4023 Float(thresClip), 4024 uInt(nIterClip), 4025 Float(threshold), 4026 uInt(chanAvgLimit), 4027 cEdge); 4028 } else { 4029 setSpectrum(res, whichrow); 4030 } 3324 4031 3325 4032 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "autoCubicSplineBaseline()", pieceEdges, params, nClipped); … … 3327 4034 } 3328 4035 4036 //--- 4037 if (outBaselintTable) { 4038 bt->save(bltable); 4039 } 4040 //--- 4041 delete bt; 4042 3329 4043 if (outTextFile) ofs.close(); 3330 4044 … … 3334 4048 } 3335 4049 3336 std::vector<float> Scantable::doCubicSplineFitting(const std::vector<float>& data, const std::vector<bool>& mask, int nPiece, std::vector<int>& idxEdge, std::vector<float>& params, int& nClipped, float thresClip, int nIterClip, bool getResidual) 4050 std::vector<float> Scantable::doCubicSplineFitting(const std::vector<float>& data, const std::vector<bool>& mask, std::vector<int>& idxEdge, std::vector<float>& params, float& rms, std::vector<bool>& finalmask, float clipth, int clipn) 4051 { 4052 int nClipped = 0; 4053 return doCubicSplineFitting(data, mask, idxEdge.size()-1, true, idxEdge, params, rms, finalmask, nClipped, clipth, clipn); 4054 } 4055 4056 std::vector<float> Scantable::doCubicSplineFitting(const std::vector<float>& data, const std::vector<bool>& mask, int nPiece, std::vector<int>& idxEdge, std::vector<float>& params, float& rms, std::vector<bool>& finalmask, float clipth, int clipn) 4057 { 4058 int nClipped = 0; 4059 return doCubicSplineFitting(data, mask, nPiece, false, idxEdge, params, rms, finalmask, nClipped, clipth, clipn); 4060 } 4061 4062 std::vector<float> Scantable::doCubicSplineFitting(const std::vector<float>& data, const std::vector<bool>& mask, int nPiece, bool useGivenPieceBoundary, std::vector<int>& idxEdge, std::vector<float>& params, float& rms, std::vector<bool>& finalMask, int& nClipped, float thresClip, int nIterClip, bool getResidual) 3337 4063 { 3338 4064 if (data.size() != mask.size()) { … … 3344 4070 3345 4071 int nChan = data.size(); 4072 4073 finalMask.clear(); 4074 finalMask.resize(nChan); 4075 3346 4076 std::vector<int> maskArray(nChan); 3347 4077 std::vector<int> x(nChan); … … 3353 4083 j++; 3354 4084 } 4085 finalMask[i] = mask[i]; 3355 4086 } 3356 4087 int initNData = j; … … 3362 4093 int nElement = (int)(floor(floor((double)(initNData/nPiece))+0.5)); 3363 4094 std::vector<double> invEdge(nPiece-1); 3364 idxEdge[0] = x[0]; 4095 4096 if (useGivenPieceBoundary) { 4097 if ((int)idxEdge.size() != nPiece+1) { 4098 throw(AipsError("pieceEdge.size() must be equal to nPiece+1.")); 4099 } 4100 } else { 4101 idxEdge.clear(); 4102 idxEdge.resize(nPiece+1); 4103 idxEdge[0] = x[0]; 4104 } 3365 4105 for (int i = 1; i < nPiece; ++i) { 3366 4106 int valX = x[nElement*i]; 3367 idxEdge[i] = valX; 4107 if (!useGivenPieceBoundary) { 4108 idxEdge[i] = valX; 4109 } 3368 4110 invEdge[i-1] = 1.0/(double)valX; 3369 4111 } 3370 idxEdge[nPiece] = x[initNData-1]+1; 4112 if (!useGivenPieceBoundary) { 4113 idxEdge[nPiece] = x[initNData-1]+1; 4114 } 4115 4116 params.clear(); 4117 params.resize(nPiece*4); 3371 4118 3372 4119 int nData = initNData; … … 3395 4142 // identity matrix (right). 3396 4143 // the right part is used to calculate the inverse matrix of the left part. 4144 3397 4145 double xMatrix[nDOF][2*nDOF]; 3398 4146 double zMatrix[nDOF]; … … 3496 4244 } 3497 4245 } 4246 3498 4247 //compute a vector y which consists of the coefficients of the best-fit spline curves 3499 4248 //(a0,a1,a2,a3(,b3,c3,...)), namely, the ones for the leftmost piece and the ones of … … 3545 4294 } 3546 4295 } 4296 3547 4297 if (idxEdge[nPiece] < nChan) { 3548 4298 int n = idxEdge[nPiece]-1; … … 3562 4312 } 3563 4313 4314 double stdDev = 0.0; 4315 for (int i = 0; i < nChan; ++i) { 4316 stdDev += residual[i]*residual[i]*(double)maskArray[i]; 4317 } 4318 stdDev = sqrt(stdDev/(double)nData); 4319 rms = (float)stdDev; 4320 3564 4321 if ((nClip == nIterClip) || (thresClip <= 0.0)) { 3565 4322 break; 3566 4323 } else { 3567 double stdDev = 0.0;3568 for (int i = 0; i < nChan; ++i) {3569 stdDev += residual[i]*residual[i]*(double)maskArray[i];3570 }3571 stdDev = sqrt(stdDev/(double)nData);3572 4324 3573 4325 double thres = stdDev * thresClip; … … 3576 4328 if (abs(residual[i]) >= thres) { 3577 4329 maskArray[i] = 0; 4330 finalMask[i] = false; 3578 4331 } 3579 4332 if (maskArray[i] > 0) { … … 3586 4339 nData = newNData; 3587 4340 } 4341 3588 4342 } 3589 4343 } … … 3605 4359 } 3606 4360 3607 void Scantable::selectWaveNumbers(const int whichrow, const std::vector<bool>& chanMask, const bool applyFFT, const std::string& fftMethod, const std::string& fftThresh, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, std::vector<int>& nWaves) 3608 { 4361 void Scantable::selectWaveNumbers(const int whichrow, const std::vector<bool>& chanMask, const std::string& fftInfo, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, std::vector<int>& nWaves) 4362 //void Scantable::selectWaveNumbers(const int whichrow, const std::vector<bool>& chanMask, const bool applyFFT, const std::string& fftMethod, const std::string& fftThresh, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, std::vector<int>& nWaves) 4363 { 4364 bool applyFFT; 4365 std::string fftMethod; 4366 std::string fftThresh; 4367 3609 4368 nWaves.clear(); 3610 4369 4370 parseFFTInfo(fftInfo, applyFFT, fftMethod, fftThresh); 3611 4371 if (applyFFT) { 3612 4372 string fftThAttr; … … 3618 4378 3619 4379 addAuxWaveNumbers(whichrow, addNWaves, rejectNWaves, nWaves); 4380 } 4381 4382 void Scantable::parseFFTInfo(const std::string& fftInfo, bool& applyFFT, std::string& fftMethod, std::string& fftThresh) 4383 { 4384 istringstream iss(fftInfo); 4385 std::string tmp; 4386 std::vector<string> res; 4387 while (getline(iss, tmp, ',')) { 4388 res.push_back(tmp); 4389 } 4390 if (res.size() < 3) { 4391 throw(AipsError("wrong value in 'fftinfo' parameter")) ; 4392 } 4393 applyFFT = (res[0] == "true"); 4394 fftMethod = res[1]; 4395 fftThresh = res[2]; 3620 4396 } 3621 4397 … … 3702 4478 { 3703 4479 std::vector<int> tempAddNWaves, tempRejectNWaves; 4480 tempAddNWaves.clear(); 4481 tempRejectNWaves.clear(); 4482 3704 4483 for (uInt i = 0; i < addNWaves.size(); ++i) { 3705 4484 tempAddNWaves.push_back(addNWaves[i]); … … 3745 4524 void Scantable::setWaveNumberListUptoNyquistFreq(const int whichrow, std::vector<int>& nWaves) 3746 4525 { 3747 if ((nWaves.size() == 2)&&(nWaves[1] == -999)) { 3748 int val = nWaves[0]; 3749 int nyquistFreq = nchan(getIF(whichrow))/2+1; 3750 nWaves.clear(); 3751 if (val > nyquistFreq) { // for safety, at least nWaves contains a constant; CAS-3759 3752 nWaves.push_back(0); 3753 } 3754 while (val <= nyquistFreq) { 3755 nWaves.push_back(val); 3756 val++; 3757 } 3758 } 3759 } 3760 3761 void Scantable::sinusoidBaseline(const std::vector<bool>& mask, const bool applyFFT, const std::string& fftMethod, const std::string& fftThresh, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 4526 int val = nWaves[0]; 4527 int nyquistFreq = nchan(getIF(whichrow))/2+1; 4528 nWaves.clear(); 4529 if (val > nyquistFreq) { // for safety, at least nWaves contains a constant; CAS-3759 4530 nWaves.push_back(0); 4531 } 4532 while (val <= nyquistFreq) { 4533 nWaves.push_back(val); 4534 val++; 4535 } 4536 } 4537 4538 void Scantable::sinusoidBaseline(const std::vector<bool>& mask, const std::string& fftInfo, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, float thresClip, int nIterClip, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 3762 4539 { 3763 4540 try { … … 3780 4557 } 3781 4558 3782 //Fitter fitter = Fitter();3783 //fitter.setExpression("sinusoid", nWaves);3784 //fitter.setIterClipping(thresClip, nIterClip);4559 bool showProgress; 4560 int minNRow; 4561 parseProgressInfo(progressInfo, showProgress, minNRow); 3785 4562 3786 4563 int nRow = nrow(); 3787 4564 std::vector<bool> chanMask; 3788 4565 std::vector<int> nWaves; 3789 3790 bool showProgress; 3791 int minNRow; 3792 parseProgressInfo(progressInfo, showProgress, minNRow); 4566 std::vector<bool> finalChanMask; 4567 float rms; 4568 4569 //--- 4570 bool outBaselintTable = (bltable != ""); 4571 STBaselineTable* bt = new STBaselineTable(*this); 4572 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 4573 Vector<Double> timeSecCol = tcol->getColumn(); 4574 //--- 3793 4575 3794 4576 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 4577 std::vector<float> sp = getSpectrum(whichrow); 3795 4578 chanMask = getCompositeChanMask(whichrow, mask); 3796 selectWaveNumbers(whichrow, chanMask, applyFFT, fftMethod, fftThresh, addNWaves, rejectNWaves, nWaves); 3797 3798 //FOR DEBUGGING------------ 3799 /* 3800 if (whichrow < 0) {// == nRow -1) { 3801 cout << "+++ i=" << setw(3) << whichrow << ", IF=" << setw(2) << getIF(whichrow); 3802 if (applyFFT) { 3803 cout << "[ "; 3804 for (uInt j = 0; j < nWaves.size(); ++j) { 3805 cout << nWaves[j] << ", "; 3806 } 3807 cout << " ] " << endl; 3808 } 3809 cout << flush; 3810 } 3811 */ 3812 //------------------------- 3813 3814 //fitBaseline(chanMask, whichrow, fitter); 3815 //setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 4579 selectWaveNumbers(whichrow, chanMask, fftInfo, addNWaves, rejectNWaves, nWaves); 4580 3816 4581 std::vector<float> params; 3817 4582 int nClipped = 0; 3818 std::vector<float> res = doSinusoidFitting(getSpectrum(whichrow), chanMask, nWaves, params, nClipped, thresClip, nIterClip, getResidual); 3819 setSpectrum(res, whichrow); 3820 // 4583 std::vector<float> res = doSinusoidFitting(sp, chanMask, nWaves, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 4584 4585 //--- 4586 if (outBaselintTable) { 4587 uInt nparam = nWaves.size(); 4588 Vector<Int> fparam(nparam); 4589 for (uInt i = 0; i < nparam; ++i) { 4590 fparam[i] = nWaves[i]; 4591 } 4592 std::vector<int> finalMaskList0; 4593 finalMaskList0.clear(); 4594 for (uInt i = 0; i < finalChanMask.size(); ++i) { 4595 if (finalChanMask[i]) { 4596 if ((i == 0)||(i == finalChanMask.size()-1)) { 4597 finalMaskList0.push_back(i); 4598 } else { 4599 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 4600 finalMaskList0.push_back(i); 4601 } 4602 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 4603 finalMaskList0.push_back(i); 4604 } 4605 } 4606 } 4607 } 4608 Vector<uInt> finalMaskList(finalMaskList0.size()); 4609 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 4610 finalMaskList[i] = (uInt)finalMaskList0[i]; 4611 } 4612 4613 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 4614 uInt(0), 4615 timeSecCol[whichrow], 4616 Bool(true), 4617 STBaselineFunc::Sinusoid, 4618 fparam, 4619 Vector<Float>(), 4620 finalMaskList, 4621 Vector<Float>(params), 4622 Float(rms), 4623 uInt(sp.size()), 4624 Float(thresClip), 4625 uInt(nIterClip), 4626 Float(0.0), 4627 uInt(0), 4628 Vector<uInt>()); 4629 } else { 4630 setSpectrum(res, whichrow); 4631 } 4632 //--- 3821 4633 3822 4634 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "sinusoidBaseline()", params, nClipped); … … 3824 4636 } 3825 4637 4638 //--- 4639 if (outBaselintTable) { 4640 bt->save(bltable); 4641 } 4642 //--- 4643 delete bt; 4644 3826 4645 if (outTextFile) ofs.close(); 3827 4646 … … 3831 4650 } 3832 4651 3833 void Scantable::autoSinusoidBaseline(const std::vector<bool>& mask, const bool applyFFT, const std::string& fftMethod, const std::string& fftThresh, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile) 4652 void Scantable::autoSinusoidBaseline(const std::vector<bool>& mask, const std::string& fftInfo, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 4653 //void Scantable::autoSinusoidBaseline(const std::vector<bool>& mask, const bool applyFFT, const std::string& fftMethod, const std::string& fftThresh, const std::vector<int>& addNWaves, const std::vector<int>& rejectNWaves, float thresClip, int nIterClip, const std::vector<int>& edge, float threshold, int chanAvgLimit, bool getResidual, const std::string& progressInfo, const bool outLogger, const std::string& blfile, const std::string& bltable) 3834 4654 { 3835 4655 try { … … 3852 4672 } 3853 4673 3854 //Fitter fitter = Fitter(); 3855 //fitter.setExpression("sinusoid", nWaves); 3856 //fitter.setIterClipping(thresClip, nIterClip); 4674 bool showProgress; 4675 int minNRow; 4676 parseProgressInfo(progressInfo, showProgress, minNRow); 4677 4678 int minEdgeSize = getIFNos().size()*2; 4679 STLineFinder lineFinder = STLineFinder(); 4680 lineFinder.setOptions(threshold, 3, chanAvgLimit); 3857 4681 3858 4682 int nRow = nrow(); 3859 4683 std::vector<bool> chanMask; 3860 4684 std::vector<int> nWaves; 3861 3862 int minEdgeSize = getIFNos().size()*2; 3863 STLineFinder lineFinder = STLineFinder(); 3864 lineFinder.setOptions(threshold, 3, chanAvgLimit); 3865 3866 bool showProgress; 3867 int minNRow; 3868 parseProgressInfo(progressInfo, showProgress, minNRow); 4685 std::vector<bool> finalChanMask; 4686 float rms; 4687 4688 //--- 4689 bool outBaselintTable = (bltable != ""); 4690 STBaselineTable* bt = new STBaselineTable(*this); 4691 ROScalarColumn<Double> *tcol = new ROScalarColumn<Double>(table_, "TIME"); 4692 Vector<Double> timeSecCol = tcol->getColumn(); 4693 //--- 3869 4694 3870 4695 for (int whichrow = 0; whichrow < nRow; ++whichrow) { 3871 3872 //------------------------------------------------------- 3873 //chanMask = getCompositeChanMask(whichrow, mask, edge, minEdgeSize, lineFinder); 3874 //------------------------------------------------------- 4696 std::vector<float> sp = getSpectrum(whichrow); 4697 //-- composote given channel mask and flagtra -------- 3875 4698 int edgeSize = edge.size(); 3876 4699 std::vector<int> currentEdge; 4700 currentEdge.clear(); 3877 4701 if (edgeSize >= 2) { 3878 4702 int idx = 0; … … 3888 4712 throw(AipsError("Wrong length of edge element")); 3889 4713 } 3890 lineFinder.setData( getSpectrum(whichrow));4714 lineFinder.setData(sp); 3891 4715 lineFinder.findLines(getCompositeChanMask(whichrow, mask), currentEdge, whichrow); 3892 4716 chanMask = lineFinder.getMask(); 3893 //------------------------------------------------------- 3894 3895 selectWaveNumbers(whichrow, chanMask, applyFFT, fftMethod, fftThresh, addNWaves, rejectNWaves, nWaves); 3896 3897 //fitBaseline(chanMask, whichrow, fitter); 3898 //setSpectrum((getResidual ? fitter.getResidual() : fitter.getFit()), whichrow); 4717 //-- composote given channel mask and flagtra END -------- 4718 selectWaveNumbers(whichrow, chanMask, fftInfo, addNWaves, rejectNWaves, nWaves); 4719 3899 4720 std::vector<float> params; 3900 4721 int nClipped = 0; 3901 std::vector<float> res = doSinusoidFitting(getSpectrum(whichrow), chanMask, nWaves, params, nClipped, thresClip, nIterClip, getResidual); 3902 setSpectrum(res, whichrow); 3903 // 4722 std::vector<float> res = doSinusoidFitting(sp, chanMask, nWaves, params, rms, finalChanMask, nClipped, thresClip, nIterClip, getResidual); 4723 4724 //--- 4725 if (outBaselintTable) { 4726 uInt nparam = nWaves.size(); 4727 Vector<Int> fparam(nparam); 4728 for (uInt i = 0; i < nparam; ++i) { 4729 fparam[i] = nWaves[i]; 4730 } 4731 std::vector<int> finalMaskList0; 4732 finalMaskList0.clear(); 4733 for (uInt i = 0; i < finalChanMask.size(); ++i) { 4734 if (finalChanMask[i]) { 4735 if ((i == 0)||(i == finalChanMask.size()-1)) { 4736 finalMaskList0.push_back(i); 4737 } else { 4738 if ((finalChanMask[i])&&(!finalChanMask[i-1])) { 4739 finalMaskList0.push_back(i); 4740 } 4741 if ((finalChanMask[i])&&(!finalChanMask[i+1])) { 4742 finalMaskList0.push_back(i); 4743 } 4744 } 4745 } 4746 } 4747 Vector<uInt> finalMaskList(finalMaskList0.size()); 4748 for (uInt i = 0; i < finalMaskList0.size(); ++i) { 4749 finalMaskList[i] = (uInt)finalMaskList0[i]; 4750 } 4751 4752 Vector<uInt> cEdge(currentEdge.size()); 4753 for (uInt i = 0; i < currentEdge.size(); ++i) { 4754 cEdge[i] = currentEdge[i]; 4755 } 4756 4757 bt->appenddata(uInt(getScan(whichrow)), uInt(getCycle(whichrow)), uInt(getBeam(whichrow)), uInt(getIF(whichrow)), uInt(getPol(whichrow)), 4758 uInt(0), 4759 timeSecCol[whichrow], 4760 Bool(true), 4761 STBaselineFunc::Sinusoid, 4762 fparam, 4763 Vector<Float>(), 4764 finalMaskList, 4765 Vector<Float>(params), 4766 Float(rms), 4767 uInt(sp.size()), 4768 Float(thresClip), 4769 uInt(nIterClip), 4770 Float(threshold), 4771 uInt(chanAvgLimit), 4772 cEdge); 4773 } else { 4774 setSpectrum(res, whichrow); 4775 } 4776 //--- 3904 4777 3905 4778 outputFittingResult(outLogger, outTextFile, csvFormat, chanMask, whichrow, coordInfo, hasSameNchan, ofs, "autoSinusoidBaseline()", params, nClipped); … … 3907 4780 } 3908 4781 4782 //--- 4783 if (outBaselintTable) { 4784 bt->save(bltable); 4785 } 4786 //--- 4787 delete bt; 4788 3909 4789 if (outTextFile) ofs.close(); 3910 4790 … … 3914 4794 } 3915 4795 3916 std::vector<float> Scantable::doSinusoidFitting(const std::vector<float>& data, const std::vector<bool>& mask, const std::vector<int>& waveNumbers, std::vector<float>& params, int& nClipped, float thresClip, int nIterClip, bool getResidual) 4796 std::vector<float> Scantable::doSinusoidFitting(const std::vector<float>& data, const std::vector<bool>& mask, const std::vector<int>& waveNumbers, std::vector<float>& params, float& rms, std::vector<bool>& finalmask, float clipth, int clipn) 4797 { 4798 int nClipped = 0; 4799 return doSinusoidFitting(data, mask, waveNumbers, params, rms, finalmask, nClipped, clipth, clipn); 4800 } 4801 4802 std::vector<float> Scantable::doSinusoidFitting(const std::vector<float>& data, const std::vector<bool>& mask, const std::vector<int>& waveNumbers, std::vector<float>& params, float& rms, std::vector<bool>& finalMask, int& nClipped, float thresClip, int nIterClip, bool getResidual) 3917 4803 { 3918 4804 if (data.size() != mask.size()) { … … 3939 4825 3940 4826 int nChan = data.size(); 4827 4828 finalMask.clear(); 4829 finalMask.resize(nChan); 4830 3941 4831 std::vector<int> maskArray; 3942 4832 std::vector<int> x; … … 3946 4836 x.push_back(i); 3947 4837 } 4838 finalMask[i] = mask[i]; 3948 4839 } 3949 4840 … … 4086 4977 } 4087 4978 4979 double stdDev = 0.0; 4980 for (int i = 0; i < nChan; ++i) { 4981 stdDev += residual[i]*residual[i]*(double)maskArray[i]; 4982 } 4983 stdDev = sqrt(stdDev/(double)nData); 4984 rms = (float)stdDev; 4985 4088 4986 if ((nClip == nIterClip) || (thresClip <= 0.0)) { 4089 4987 break; 4090 4988 } else { 4091 double stdDev = 0.0; 4092 for (int i = 0; i < nChan; ++i) { 4093 stdDev += residual[i]*residual[i]*(double)maskArray[i]; 4094 } 4095 stdDev = sqrt(stdDev/(double)nData); 4096 4989 4097 4990 double thres = stdDev * thresClip; 4098 4991 int newNData = 0; … … 4100 4993 if (abs(residual[i]) >= thres) { 4101 4994 maskArray[i] = 0; 4995 finalMask[i] = false; 4102 4996 } 4103 4997 if (maskArray[i] > 0) { … … 4110 5004 nData = newNData; 4111 5005 } 5006 4112 5007 } 4113 5008 }
Note:
See TracChangeset
for help on using the changeset viewer.