www.webdeveloper.com
Page 1 of 2 12 LastLast
Results 1 to 15 of 17

Thread: Real-time spectrogram

  1. #1
    Join Date
    Feb 2017
    Posts
    15

    Real-time spectrogram

    I'm trying to create real-time spectrograms using javascript. but the response keeps getting delayed. Is it because of computational limitations of the browser/my computer/the webkit or is there something in my code that is messing things up?

    var yAxis=0;
    var n=0;
    var canvas = document.getElementById("can");
    var ctx = canvas.getContext("2d");
    var node;
    var input;
    var ad;
    var bol=true;

    var context = new AudioContext();
    navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia;
    var analyzer = context.createAnalyser();
    var frequencyData;
    ///
    var hi = context.createBiquadFilter();
    hi.type = "highpass";
    hi.frequency=20000;
    hi.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi2 = context.createBiquadFilter();
    hi2.type = "highpass";
    hi2.frequency=10000;
    hi2.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi3 = context.createBiquadFilter();
    hi3.type = "highpass";
    hi3.frequency=5000;
    hi3.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi4 = context.createBiquadFilter();
    hi4.type = "highpass";
    hi4.frequency=2500;
    hi4.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi5 = context.createBiquadFilter();
    hi5.type = "highpass";
    hi5.frequency=1250;
    hi5.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi6 = context.createBiquadFilter();
    hi6.type = "highpass";
    hi6.frequency=800;
    hi6.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi7 = context.createBiquadFilter();
    hi7.type = "highpass";
    hi7.frequency=10000;
    hi7.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi8 = context.createBiquadFilter();
    hi8.type = "highpass";
    hi8.frequency=10000;
    hi8.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi9 = context.createBiquadFilter();
    hi9.type = "highpass";
    hi9.frequency=10000;
    hi9.Q.setTargetAtTime(1, context.currentTime + 0.1, 0.1);
    var hi10 = context.createBiquadFilter();
    hi10.type = "peaking";
    hi10.frequency=600;
    hi10.gain.setTargetAtTime(-90, context.currentTime + 0.1, 0.1);
    var hi11 = context.createBiquadFilter();
    hi11.type = "peaking";
    hi11.frequency=100;
    hi11.gain.setTargetAtTime(1, context.currentTime + 1, 0.5);
    ///


    function gra(){



    navigator.getUserMedia({ audio: true }, spect,error);

    analyzer.fftSize = 512;
    bol=true;
    node = context.createScriptProcessor(1024, 1, 1);

    hi10.connect(hi);
    hi.connect(hi2)
    hi2.connect(hi3);
    hi3.connect(hi4);
    hi4.connect(hi5);
    hi5.connect(hi6);
    hi6.connect(hi7);
    hi7.connect(hi8);
    hi8.connect(hi9);
    hi9.connect(hi11);
    hi11.connect(analyzer);
    analyzer.connect(node);
    node.connect(context.destination);


    }

    function spect (stream) {
    input = context.createMediaStreamSource(stream);
    input.connect(hi10);
    if (bol==true){

    if (yAxis>1599){
    yAxis=0;
    }
    ctx.strokeStyle = "#ffffff";
    ctx.moveTo(yAxis+1,512);
    ctx.lineTo(yAxis+1,0);
    ctx.stroke();
    ctx.beginPath();
    n=0;
    ad=0;

    frequencyData = new Uint8Array(analyzer.frequencyBinCount);
    analyzer.getByteFrequencyData(frequencyData);

    while (n<50) {

    if (n==49){

    yAxis=yAxis+1;


    setTimeout(function(){
    spect(stream)
    },15)
    break;
    } else {

    if (n<6){
    if (frequencyData[n]>137){
    ctx.strokeStyle = "#000000";
    } else{
    ctx.strokeStyle = "#"+(155-frequencyData[n]).toString(16)+(155-frequencyData[n]).toString(16)+(155-frequencyData[n]).toString(16);
    }
    } else if (n>16){
    if (frequencyData[n]>202){
    ctx.strokeStyle = "#000000";
    } else {
    ctx.strokeStyle = "#"+(220 - frequencyData[n]).toString(16)+(220 - frequencyData[n]).toString(16)+(220 - frequencyData[n]).toString(16);
    }
    } else{
    if (frequencyData[n]>217){
    ctx.strokeStyle = "#000000";
    } else {
    ctx.strokeStyle = "#"+(235 - frequencyData[n]).toString(16)+(235 - frequencyData[n]).toString(16)+(235 - frequencyData[n]).toString(16);
    }
    }
    ctx.moveTo(yAxis,512-ad);
    ctx.lineTo(yAxis,512-ad+8);
    ctx.stroke();
    ctx.beginPath();
    n=n+1;
    ad=ad+8;

    }
    ;
    }

    }
    }

    function error () {
    alert("The action cannot be completed!")
    }

    function stop(){
    bol=false;

    }

  2. #2
    Join Date
    Dec 2012
    Posts
    1,628
    Cool stuff! I increased the interval to 50ms and on my computer the cursor is running smoothly, although the computer is not a very powerful one. The summary of the performance analysis in Opera shows a lot of idle time:
    Recording time: 14.15s
    Idle time: 13.165s
    (Unfortunately I didn't succeed in uploading a screenshot. The CSS of this forum seems to be somewhat broken.)

  3. #3
    Join Date
    Feb 2017
    Posts
    15
    Thanks for your quick reply, but the problem seems to appear when the yAxis variable resets to 0. Then it gets delayed regardless of the interval.

  4. #4
    Join Date
    Dec 2005
    Location
    TX
    Posts
    7,943
    Quote Originally Posted by Sempervivum View Post
    Cool stuff! I increased the interval to 50ms and on my computer the cursor is running smoothly, although the computer is not a very powerful one. The summary of the performance analysis in Opera shows a lot of idle time:
    Recording time: 14.15s
    Idle time: 13.165s
    (Unfortunately I didn't succeed in uploading a screenshot. The CSS of this forum seems to be somewhat broken.)

    What did you do to even get an output?
    I tried running the script after adding a canvas to the HTML and get nothing!

  5. #5
    Join Date
    Feb 2017
    Posts
    15
    Did you make sure you named the canvas "can"?

  6. #6
    Join Date
    Dec 2012
    Posts
    1,628
    Additionally the script needs a source for recording. However when there is none you should get an alert.
    As I have no micro I used the stereo mix.

    the problem seems to appear when the yAxis variable resets to 0. Then it gets delayed regardless of the interval.
    Not for me: When reaching the end of the bar at the right it skips to the beginning at the left und starts moving without delay. However at the beginning the bar was very large so that some section at the right wasn't visible. When the cursor disappeared at the right it seemed that it stopped moving but this was an illusion as it kept moving but was not visible. Therefore I reduced the size of the bar to something like 500px.
    Last edited by Sempervivum; 02-10-2018 at 11:00 AM.

  7. #7
    Join Date
    Dec 2005
    Location
    TX
    Posts
    7,943
    Quote Originally Posted by pabloma2002 View Post
    Did you make sure you named the canvas "can"?
    Yes.

  8. #8
    Join Date
    Mar 2011
    Posts
    128
    I haven't tested it in a webkit browser, but when I tried it in vivaldi the back and refresh button stopped responding after a while. After I took context.createMediaStreamSource(stream) out of the spect() function and placed it in the gra() function so it only creates one instance, my browser stayed responsive.
    I reorganised most of the code
    Code:
    <span></span>
    <canvas id="can" width="1600" style="border: 1px solid #000"></canvas>
    
    <script>
    
    var myAudio = document.querySelector('audio');
    var dbg = document.querySelector('span');
    var canvas = document.getElementById("can");
    	
    var ctx = canvas.getContext("2d");
    var yAxis = 0;
    
    var bol=true;
    
    var context = new AudioContext();
    var analyzer = context.createAnalyser();
    
    gra();
    
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
    
    function makeBiquadFilter(ctx, type, frq, audioParameter, t0, t1, t2) {
     var bqf = ctx.createBiquadFilter();
     bqf.type = type;
     bqf.frequency = frq;
     bqf[audioParameter].setTargetAtTime(t0, ctx.currentTime + t1, t2);
     return bqf;
    };
    
    function getGray(l) {
     return 'rgb('+[l, l, l]+')'; // 'hsl(0, 0%, '+ l / 2.55 +'%)';
    };
    
    function gra(){
     var node = context.createScriptProcessor(1024, 1, 1);
    
     var hi = makeBiquadFilter(context, "highpass", 20000, 'Q', 1, .1, .1);
     var hi2 = makeBiquadFilter(context, "highpass", 10000, 'Q', 1, .1, .1);
     var hi3 = makeBiquadFilter(context, "highpass", 5000, 'Q', 1, .1, .1);
     var hi4 = makeBiquadFilter(context, "highpass", 2500, 'Q', 1, .1, .1);
     var hi5 = makeBiquadFilter(context, "highpass", 1250, 'Q', 1, .1, .1);
     var hi6 = makeBiquadFilter(context, "highpass", 800, 'Q', 1, .1, .1);
     var hi7 = makeBiquadFilter(context, "highpass", 10000, 'Q', 1, .1, .1);
     var hi8 = makeBiquadFilter(context, "highpass", 10000, 'Q', 1, .1, .1);
     var hi9 = makeBiquadFilter(context, "highpass", 10000, 'Q', 1, .1, .1);
     var hi10 = makeBiquadFilter(context, "peaking", 600, 'gain', -90, .1, .1);
     var hi11 = makeBiquadFilter(context, "peaking", 100, 'gain', 1, 1, .5);
    
     analyzer.fftSize = 512;
     bol = true;
    
     hi10.connect(hi);
     hi.connect(hi2)
     hi2.connect(hi3);
     hi3.connect(hi4);
     hi4.connect(hi5);
     hi5.connect(hi6);
     hi6.connect(hi7);
     hi7.connect(hi8);
     hi8.connect(hi9);
     hi9.connect(hi11);
     hi11.connect(analyzer);
     node.connect(context.destination);
    
     t0 = performance.now();
    
     // navigator.getUserMedia({ audio: true }, spect, error);
    
     navigator.mediaDevices.getUserMedia({ audio: true }).then(
      function(stream){
       var input = context.createMediaStreamSource(stream);
       input.connect(hi10);
       spect();
     }).catch(error);
    };
    
    function spect() {
     if(bol == false) {
      return;
     }
    
     dbg.innerText = parseInt((t1 = performance.now()) - t0);
     t0 = t1;
    
     var frqN;
     var n = 0;
     var ad = 0;
    
     ctx.strokeStyle = "#ffffff";
     ctx.moveTo(yAxis + 1, 512);
     ctx.lineTo(yAxis + 1, 0);
     ctx.stroke();
     ctx.beginPath();
    
     var frequencyData = new Uint8Array(analyzer.frequencyBinCount);
     analyzer.getByteFrequencyData(frequencyData);
    
     while(n < 49) {
      frqN = frequencyData[n];
       ctx.strokeStyle = "#000000";
    
      if(n < 6 && frqN <= 137) {
         ctx.strokeStyle = getGray(155 - frqN);
      }
      else if(n > 16  && frqN <= 202) {
         ctx.strokeStyle = getGray(220 - frqN);
      }
      else if(frqN <= 217) {
         ctx.strokeStyle = getGray(235 - frqN);
      }
    
      ctx.moveTo(yAxis, 512 - ad);
      ctx.lineTo(yAxis, 512 - ad + 8);
      ctx.stroke();
      ctx.beginPath();
      n++;	
      ad += 8;
     }//while
    
     if(++yAxis > 1599) {
      yAxis = 0;
     }
     window.requestAnimationFrame(spect);
    };
    
    function error (err) {
     alert("The action cannot be completed!\n" + err)
    }
    
    </script>

  9. #9
    Join Date
    Mar 2007
    Location
    localhost
    Posts
    5,868
    "Real Time" and the "Reality" are two different things.

    You have to remember that in all systems, you have latency, that is not only in the system itself, but the code execution as well as the data stream input and any calculations.

    Also, setTimeout will introduce its own latency in to the mix.

    Where you will be calling the same object more than once, you would be better to use the setInterval timer function instead of setTimeout.
    --> JavaScript Frameworks like JQuery, Angular, Node <--
    ... and please remember to wrap code with forum BBCode tags:-

    [CODE]...[/CODE] [HTML]...[/HTML] [PHP]...[/PHP]

    If you can't think outside the box, you will be trapped forever with no escape...

  10. #10
    Join Date
    Feb 2017
    Posts
    15
    The problem is not setTimeout, there is something else I'm missing, because everything works well until the yAxis variable resets to "0".

  11. #11
    Join Date
    Mar 2007
    Location
    localhost
    Posts
    5,868
    setTimeout introduces latency in calling the functions, that will be the first place I look at first.

    TBH you are asking too much of JavaScript in respect to performing, if you really want something like this, then I suggest a compiled language.
    --> JavaScript Frameworks like JQuery, Angular, Node <--
    ... and please remember to wrap code with forum BBCode tags:-

    [CODE]...[/CODE] [HTML]...[/HTML] [PHP]...[/PHP]

    If you can't think outside the box, you will be trapped forever with no escape...

  12. #12
    Join Date
    Feb 2017
    Posts
    15
    I tried adding the line: document.getElementById("txt").innerHTML=yAxis;
    to see what happened and in fact the drawing function is not delayed, its the microphone response that is delayed, but for some reason exactly when the yAxis variable resets to"0".

  13. #13
    Join Date
    Mar 2007
    Location
    localhost
    Posts
    5,868
    Like I have said, if you are calling something at a regular interval, you set up ONCE ONLY a setInterval timer which will call the allocated function at that period in time until it is cleared with the clearInterval() call.

    With setTimeout, you have to keep setting them and they take time to set, run and then call the function.

    So if you say a modest 10ms delay in each setup and call, this means that your latency with setInterval is limited to 10ms... TOTAL...

    whereas you setTimeout will add (compounding the problem) by 10ms each iteration of the setup and function call.

    So if you are calling the routine every 50ms, its actually happening every 60ms because of the latency you introduced to start with.

    The amount of latency will depend on the computer, its ram, the resources at its disposal... so what runs fine on your computer may not work at all on another.
    --> JavaScript Frameworks like JQuery, Angular, Node <--
    ... and please remember to wrap code with forum BBCode tags:-

    [CODE]...[/CODE] [HTML]...[/HTML] [PHP]...[/PHP]

    If you can't think outside the box, you will be trapped forever with no escape...

  14. #14
    Join Date
    Mar 2011
    Posts
    128
    Did you try my suggestion taking .createMediaStreamSource() out of loop?

  15. #15
    Join Date
    Feb 2017
    Posts
    15
    Quote Originally Posted by Kever View Post
    Did you try my suggestion taking .createMediaStreamSource() out of loop?
    The problem is that I need the "stream" variable.

Thread Information

Users Browsing this Thread

There are currently 1 users browsing this thread. (0 members and 1 guests)

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •  
HTML5 Development Center

"

"

X vBulletin 4.2.2 Debug Information

  • Page Generation 0.11756 seconds
  • Memory Usage 3,029KB
  • Queries Executed 15 (?)
More Information
Template Usage (37):
  • (1)SHOWTHREAD
  • (1)ad_footer_end
  • (1)ad_footer_start
  • (1)ad_global_above_footer
  • (1)ad_global_below_navbar
  • (1)ad_global_header1
  • (1)ad_global_header2
  • (1)ad_navbar_below
  • (1)ad_showthread_firstpost_sig
  • (1)ad_showthread_firstpost_start
  • (1)ad_thread_first_post_content
  • (1)ad_thread_last_post_content
  • (1)bbcode_code
  • (4)bbcode_quote
  • (1)footer
  • (1)forumjump
  • (1)forumrules
  • (1)gobutton
  • (1)header
  • (1)headinclude
  • (1)headinclude_bottom
  • (15)memberaction_dropdown
  • (1)navbar
  • (4)navbar_link
  • (1)navbar_moderation
  • (1)navbar_noticebit
  • (1)navbar_tabs
  • (2)option
  • (1)pagenav
  • (1)pagenav_curpage
  • (1)pagenav_pagelink
  • (15)postbit
  • (15)postbit_onlinestatus
  • (15)postbit_wrapper
  • (1)spacer_close
  • (1)spacer_open
  • (1)tagbit_wrapper 

Phrase Groups Available (6):
  • global
  • inlinemod
  • postbit
  • posting
  • reputationlevel
  • showthread
Included Files (26):
  • ./showthread.php
  • ./global.php
  • ./includes/class_bootstrap.php
  • ./includes/init.php
  • ./includes/class_core.php
  • ./includes/config.php
  • ./includes/functions.php
  • ./includes/functions_navigation.php
  • ./includes/class_friendly_url.php
  • ./includes/class_hook.php
  • ./includes/class_bootstrap_framework.php
  • ./vb/vb.php
  • ./vb/phrase.php
  • ./includes/functions_facebook.php
  • ./includes/functions_calendar.php
  • ./includes/functions_bigthree.php
  • ./includes/class_postbit.php
  • ./includes/class_bbcode.php
  • ./includes/functions_reputation.php
  • ./includes/functions_notice.php
  • ./packages/vbattach/attach.php
  • ./vb/types.php
  • ./vb/cache.php
  • ./vb/cache/db.php
  • ./vb/cache/observer/db.php
  • ./vb/cache/observer.php 

Hooks Called (74):
  • init_startup
  • friendlyurl_resolve_class
  • init_startup_session_setup_start
  • database_pre_fetch_array
  • database_post_fetch_array
  • init_startup_session_setup_complete
  • global_bootstrap_init_start
  • global_bootstrap_init_complete
  • cache_permissions
  • fetch_postinfo_query
  • fetch_postinfo
  • fetch_threadinfo_query
  • fetch_threadinfo
  • fetch_foruminfo
  • load_show_variables
  • load_forum_show_variables
  • global_state_check
  • global_bootstrap_complete
  • global_start
  • style_fetch
  • global_setup_complete
  • showthread_start
  • showthread_getinfo
  • strip_bbcode
  • friendlyurl_clean_fragment
  • friendlyurl_geturl
  • forumjump
  • cache_templates
  • cache_templates_process
  • template_register_var
  • template_render_output
  • fetch_template_start
  • fetch_template_complete
  • parse_templates
  • fetch_musername
  • notices_check_start
  • notices_noticebit
  • process_templates_complete
  • friendlyurl_redirect_canonical
  • showthread_post_start
  • showthread_query_postids
  • showthread_query
  • bbcode_fetch_tags
  • bbcode_create
  • showthread_postbit_create
  • postbit_factory
  • postbit_display_start
  • postbit_imicons
  • bbcode_parse_start
  • bbcode_parse_complete_precache
  • bbcode_parse_complete
  • postbit_display_complete
  • memberaction_dropdown
  • pagenav_page
  • pagenav_complete
  • tag_fetchbit_complete
  • forumrules
  • navbits
  • navbits_complete
  • build_navigation_data
  • build_navigation_array
  • check_navigation_permission
  • process_navigation_links_start
  • process_navigation_links_complete
  • set_navigation_menu_element
  • build_navigation_menudata
  • build_navigation_listdata
  • build_navigation_list
  • set_navigation_tab_main
  • set_navigation_tab_fallback
  • navigation_tab_complete
  • fb_like_button
  • showthread_complete
  • page_templates